paddlex 3.0.0b2__py3-none-any.whl → 3.0.0rc0__py3-none-any.whl

paddlex/inference/serving/schemas/video_detection.py

@@ -0,0 +1,56 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Final, List, Optional
+
+from pydantic import BaseModel
+
+from ..infra.models import PrimaryOperations
+from .shared import object_detection
+
+__all__ = [
+    "INFER_ENDPOINT",
+    "InferRequest",
+    "DetectedObject",
+    "InferResult",
+    "PRIMARY_OPERATIONS",
+]
+
+INFER_ENDPOINT: Final[str] = "/video-detection"
+
+
+class InferRequest(BaseModel):
+    video: str
+    nmsThresh: Optional[float] = None
+    scoreThresh: Optional[float] = None
+
+
+class DetectedObject(BaseModel):
+    bbox: object_detection.BoundingBox
+    categoryName: str
+    score: float
+
+
+class Frame(BaseModel):
+    index: int
+    detectedObjects: List[DetectedObject]
+
+
+class InferResult(BaseModel):
+    frames: List[Frame]
+
+
+PRIMARY_OPERATIONS: Final[PrimaryOperations] = {
+    "infer": (INFER_ENDPOINT, InferRequest, InferResult),
+}

paddlex/inference/utils/benchmark.py

@@ -20,19 +20,23 @@ from pathlib import Path
 import numpy as np
 from prettytable import PrettyTable
 
-from ...utils.flags import INFER_BENCHMARK_OUTPUT
+from ...utils.flags import INFER_BENCHMARK, INFER_BENCHMARK_OUTPUT
+from ...utils.misc import Singleton
 from ...utils import logging
 
 
-class Benchmark:
-    def __init__(self, components):
-        self._components = components
+class Benchmark(metaclass=Singleton):
+    def __init__(self):
+        self._components = {}
         self._warmup_start = None
         self._warmup_elapse = None
         self._warmup_num = None
         self._e2e_tic = None
         self._e2e_elapse = None
 
+    def attach(self, component):
+        self._components[component.name] = component
+
     def start(self):
         self._warmup_start = time.time()
         self._reset()
@@ -51,13 +55,13 @@ class Benchmark:
         if cmps is None:
             return
         for name, cmp in cmps.items():
-            if cmp.sub_cmps is not None:
-                yield from self.iterate_cmp(cmp.sub_cmps)
+            if hasattr(cmp, "benchmark"):
+                yield from self.iterate_cmp(cmp.benchmark)
             yield name, cmp
 
     def gather(self, e2e_num):
         # lazy import for avoiding circular import
-        from ..components.paddle_predictor import BasePaddlePredictor
+        from ..new_models.base import BasePaddlePredictor
 
         detail = []
         summary = {"preprocess": 0, "inference": 0, "postprocess": 0}
@@ -65,7 +69,7 @@ class Benchmark:
         for name, cmp in self._components.items():
             if isinstance(cmp, BasePaddlePredictor):
                 # TODO(gaotingquan): show by hierarchy. Now dont show xxxPredictor benchmark info to ensure mutual exclusivity between components.
-                for name, sub_cmp in cmp.sub_cmps.items():
+                for name, sub_cmp in cmp.benchmark.items():
                     times = sub_cmp.timer.logs
                     counts = len(times)
                     avg = np.mean(times) * 1000
@@ -74,6 +78,8 @@ class Benchmark:
                     summary["inference"] += total
                 op_tag = "postprocess"
             else:
+                # TODO(gaotingquan): support sub_cmps for others
+                # if hasattr(cmp, "benchmark"):
                 times = cmp.timer.logs
                 counts = len(times)
                 avg = np.mean(times) * 1000
@@ -158,20 +164,23 @@ class Benchmark:
             save_dir = Path(INFER_BENCHMARK_OUTPUT)
             save_dir.mkdir(parents=True, exist_ok=True)
             csv_data = [detail_head, *detail]
-            # csv_data.extend(detail)
             with open(Path(save_dir) / "detail.csv", "w", newline="") as file:
                 writer = csv.writer(file)
                 writer.writerows(csv_data)
 
             csv_data = [summary_head, *summary]
-            # csv_data.extend(summary)
             with open(Path(save_dir) / "summary.csv", "w", newline="") as file:
                 writer = csv.writer(file)
                 writer.writerows(csv_data)
 
 
 class Timer:
-    def __init__(self):
+    def __init__(self, component):
+        from ..new_models.base import BaseComponent
+
+        assert isinstance(component, BaseComponent)
+        benchmark.attach(component)
+        component.apply = self.watch_func(component.apply)
         self._tic = None
         self._elapses = []
 
@@ -212,3 +221,6 @@ class Timer:
     @property
     def logs(self):
         return self._elapses
+
+
+benchmark = Benchmark() if INFER_BENCHMARK else None

paddlex/inference/utils/get_pipeline_path.py

@@ -18,8 +18,9 @@ from pathlib import Path
 
 def get_pipeline_path(pipeline_name):
     # XXX: using dict class to handle all pipeline configs
+    config_subdir = "configs/pipelines"
     pipeline_path = (
-        Path(__file__).parent.parent.parent / "pipelines" / f"{pipeline_name}.yaml"
+        Path(__file__).parent.parent.parent / config_subdir / f"{pipeline_name}.yaml"
     ).resolve()
     if not Path(pipeline_path).exists():
         return None

paddlex/inference/utils/io/__init__.py

@@ -20,6 +20,7 @@ from .readers import (
     CSVReader,
     PDFReader,
     YAMLReader,
+    AudioReader,
 )
 from .writers import (
     WriterType,
@@ -30,4 +31,6 @@ from .writers import (
     HtmlWriter,
     XlsxWriter,
     YAMLWriter,
+    VideoWriter,
+    MarkdownWriter,
 )

paddlex/inference/utils/io/readers.py

@@ -21,6 +21,13 @@ from PIL import Image, ImageOps
 import pandas as pd
 import numpy as np
 import yaml
+import soundfile
+import random
+import platform
+import importlib
+
+from ....utils import logging
+
 
 __all__ = [
     "ReaderType",
@@ -29,6 +36,7 @@ __all__ = [
     "CSVReader",
     "PDFReader",
     "YAMLReader",
+    "AudioReader",
 ]
 
 
@@ -90,7 +98,7 @@ class PDFReader(_BaseReader):
         super().__init__(backend, **bk_args)
 
     def read(self, in_path):
-        return self._backend.read_file(str(in_path))
+        yield from self._backend.read_file(str(in_path))
 
     def _init_backend(self, bk_type, bk_args):
         return PDFReaderBackend(**bk_args)
@@ -152,21 +160,28 @@ class VideoReader(_GenerativeReader):
         self.st_frame_id = st_frame_id
         self.max_num_frames = max_num_frames
         self.auto_close = auto_close
+        self._fps = 0
 
     def read(self, in_path):
         """read vide file from path"""
         self._backend.set_pos(self.st_frame_id)
         gen = self._backend.read_file(str(in_path))
-        if self.num_frames is not None:
+        if self.max_num_frames is not None:
             gen = itertools.islice(gen, self.num_frames)
         yield from gen
         if self.auto_close:
             self._backend.close()
 
+    def get_fps(self):
+        """get fps"""
+        return self._backend.get_fps()
+
     def _init_backend(self, bk_type, bk_args):
         """init backend"""
         if bk_type == "opencv":
             return OpenCVVideoReaderBackend(**bk_args)
+        elif bk_type == "decord":
+            return DecordVideoReaderBackend(**bk_args)
         else:
             raise ValueError("Unsupported backend type")
 
@@ -233,15 +248,13 @@ class PDFReaderBackend(_BaseReaderBackend):
         self.mat = fitz.Matrix(zoom_x, zoom_y).prerotate(rotate)
 
     def read_file(self, in_path):
-        images = []
         for page in fitz.open(in_path):
-            pix = page.get_pixmap(matrix=self.mat, alpha=False)
-            getpngdata = pix.tobytes(output="png")
-            # decode as np.uint8
-            image_array = np.frombuffer(getpngdata, dtype=np.uint8)
-            img_cv = cv2.imdecode(image_array, cv2.IMREAD_ANYCOLOR)
-            images.append(img_cv)
-        return images
+            pixmap = page.get_pixmap(matrix=self.mat, alpha=False)
+            img_cv = np.frombuffer(pixmap.samples, dtype=np.uint8).reshape(
+                pixmap.h, pixmap.w, pixmap.n
+            )
+            img_cv = cv2.cvtColor(img_cv, cv2.COLOR_RGB2BGR)
+            yield img_cv
 
 
 class _VideoReaderBackend(_BaseReaderBackend):
@@ -262,12 +275,16 @@ class OpenCVVideoReaderBackend(_VideoReaderBackend):
     def __init__(self, **bk_args):
         super().__init__()
         self.cap_init_args = bk_args
+        self.num_seg = bk_args.get("num_seg", None)
         self._cap = None
         self._pos = 0
         self._max_num_frames = None
 
+    def get_fps(self):
+        return self._cap.get(cv2.CAP_PROP_FPS)
+
     def read_file(self, in_path):
-        """read vidio file from path"""
+        """read video file from path"""
         if self._cap is not None:
             self._cap_release()
         self._cap = self._cap_open(in_path)
@@ -277,14 +294,33 @@ class OpenCVVideoReaderBackend(_VideoReaderBackend):
 
     def _read_frames(self, cap):
         """read frames"""
-        while True:
-            ret, frame = cap.read()
-            if not ret:
-                break
-            yield frame
+        if self.num_seg:
+            queue = []
+            while True:
+                ret, frame = cap.read()
+                if not ret:
+                    break
+                queue = []
+                if (
+                    len(queue) <= 0
+                ):  # At initialization, populate queue with initial frame
+                    for i in range(self.num_seg):
+                        queue.append(frame)
+                queue.append(frame)
+                queue.pop(0)
+                yield queue.copy()
+        else:
+            while True:
+                ret, frame = cap.read()
+                if not ret:
+                    break
+                yield frame
         self._cap_release()
 
     def _cap_open(self, video_path):
+        self.cap_init_args.pop("num_seg", None)
+        self.cap_init_args.pop("seg_len", None)
+        self.cap_init_args.pop("sample_type", None)
         self._cap = cv2.VideoCapture(video_path, **self.cap_init_args)
         if not self._cap.isOpened():
             raise RuntimeError(f"Failed to open {video_path}")
@@ -305,6 +341,81 @@ class OpenCVVideoReaderBackend(_VideoReaderBackend):
             self._cap = None
 
 
+class DecordVideoReaderBackend(_VideoReaderBackend):
+    """DecordVideoReaderBackend"""
+
+    def __init__(self, **bk_args):
+        super().__init__()
+        self.cap_init_args = bk_args
+        self._cap = None
+        self._pos = 0
+        self._max_num_frames = None
+        self.num_seg = bk_args.get("num_seg", 8)
+        self.seg_len = bk_args.get("seg_len", 1)
+        self.sample_type = bk_args.get("sample_type", 1)
+        self.valid_mode = True
+        self._fps = 0
+
+        # XXX(gaotingquan): There is a confict with `paddle` when import `decord` globally.
+        try:
+            self.decord_module = importlib.import_module("decord")
+        except ModuleNotFoundError():
+            raise Exception(
+                "Please install `decord` manually, otherwise, the related model cannot work. It can be automatically installed only on `x86_64`. Refers: `https://github.com/dmlc/decord`."
+            )
+
+    def set_pos(self, pos):
+        self._pos = pos
+
+    def sample(self, frames_len, video_object):
+        frames_idx = []
+        average_dur = int(frames_len / self.num_seg)
+        for i in range(self.num_seg):
+            idx = 0
+            if not self.valid_mode:
+                if average_dur >= self.seg_len:
+                    idx = random.randint(0, average_dur - self.seg_len)
+                    idx += i * average_dur
+                elif average_dur >= 1:
+                    idx += i * average_dur
+                else:
+                    idx = i
+            else:
+                if average_dur >= self.seg_len:
+                    idx = (average_dur - 1) // 2
+                    idx += i * average_dur
+                elif average_dur >= 1:
+                    idx += i * average_dur
+                else:
+                    idx = i
+            for jj in range(idx, idx + self.seg_len):
+                frames_idx.append(int(jj % frames_len))
+        frames_select = video_object.get_batch(frames_idx)
+        # dearray_to_img
+        np_frames = frames_select.asnumpy()
+        imgs = []
+        for i in range(np_frames.shape[0]):
+            imgbuf = np_frames[i]
+            imgs.append(imgbuf)
+        return imgs
+
+    def get_fps(self):
+        return self._cap.get_avg_fps()
+
+    def read_file(self, in_path):
+        """read video file from path"""
+        self._cap = self.decord_module.VideoReader(in_path)
+        frame_len = len(self._cap)
+        if self.sample_type == "uniform":
+            sample_video = self.sample(frame_len, self._cap)
+            return sample_video
+        else:
+            return self._cap
+
+    def close(self):
+        pass
+
+
 class CSVReader(_BaseReader):
     """CSVReader"""
 
@@ -349,5 +460,41 @@ class YAMLReaderBackend(_BaseReaderBackend):
 
     def read_file(self, in_path, **kwargs):
         with open(in_path, "r", encoding="utf-8", **kwargs) as yaml_file:
-            data = yaml.safe_load(yaml_file)
+            data = yaml.load(yaml_file, Loader=yaml.FullLoader)
         return data
+
+
+class AudioReader(_BaseReader):
+    def __init__(self, backend="wav", **bk_args):
+        super().__init__(backend="wav", **bk_args)
+
+    def _init_backend(self, bk_type, bk_args):
+        """init backend"""
+        if bk_type == "wav":
+            return WAVReaderBackend(**bk_args)
+        else:
+            raise ValueError("Unsupported backend type")
+
+    def read(self, in_path):
+        audio, audio_sample_rate = self._backend.read_file(str(in_path))
+        return audio, audio_sample_rate
+
+
+class _AudioReaderBackend(_BaseReaderBackend):
+    """_AudioReaderBackend"""
+
+    pass
+
+
+class WAVReaderBackend(_AudioReaderBackend):
+    """PandasCSVReaderBackend"""
+
+    def __init__(self):
+        super().__init__()
+
+    def read_file(self, in_path):
+        """read wav file from path"""
+        audio, audio_sample_rate = soundfile.read(
+            in_path, dtype="float32", always_2d=True
+        )
+        return audio, audio_sample_rate

paddlex/inference/utils/io/writers.py

@@ -35,6 +35,8 @@ __all__ = [
     "HtmlWriter",
     "XlsxWriter",
     "YAMLWriter",
+    "VideoWriter",
+    "MarkdownWriter",
 ]
 
 
@@ -114,6 +116,28 @@ class ImageWriter(_BaseWriter):
         return WriterType.IMAGE
 
 
+class VideoWriter(_BaseWriter):
+    """VideoWriter"""
+
+    def __init__(self, backend="opencv", **bk_args):
+        super().__init__(backend=backend, **bk_args)
+
+    def write(self, out_path, obj):
+        """write"""
+        return self._backend.write_obj(str(out_path), obj)
+
+    def _init_backend(self, bk_type, bk_args):
+        """init backend"""
+        if bk_type == "opencv":
+            return OpenCVVideoWriterBackend(**bk_args)
+        else:
+            raise ValueError("Unsupported backend type")
+
+    def get_type(self):
+        """get type"""
+        return WriterType.VIDEO
+
+
 class TextWriter(_BaseWriter):
     """TextWriter"""
 
@@ -210,6 +234,28 @@ class YAMLWriter(_BaseWriter):
         return WriterType.YAML
 
 
+class MarkdownWriter(_BaseWriter):
+    """MarkdownWriter"""
+
+    def __init__(self, backend="markdown", **bk_args):
+        super().__init__(backend=backend, **bk_args)
+
+    def write(self, out_path, obj):
+        """write"""
+        return self._backend.write_obj(str(out_path), obj)
+
+    def _init_backend(self, bk_type, bk_args):
+        """init backend"""
+        if bk_type == "markdown":
+            return MarkdownWriterBackend(**bk_args)
+        else:
+            raise ValueError("Unsupported backend type")
+
+    def get_type(self):
+        """get type"""
+        return WriterType.MARKDOWN
+
+
 class _BaseWriterBackend(object):
     """_BaseWriterBackend"""
 
@@ -266,7 +312,8 @@ class OpenCVImageWriterBackend(_ImageWriterBackend):
     def _write_obj(self, out_path, obj):
         """write image object by OpenCV"""
         if isinstance(obj, Image.Image):
-            arr = np.asarray(obj)
+            # Assuming the channel order is RGB.
+            arr = np.asarray(obj)[:, :, ::-1]
         elif isinstance(obj, np.ndarray):
             arr = obj
         else:
@@ -294,6 +341,30 @@ class PILImageWriterBackend(_ImageWriterBackend):
         return img.save(out_path, format=self.format)
 
 
+class _VideoWriterBackend(_BaseWriterBackend):
+    """_VideoWriterBackend"""
+
+    pass
+
+
+class OpenCVVideoWriterBackend(_VideoWriterBackend):
+    """OpenCVImageWriterBackend"""
+
+    def _write_obj(self, out_path, obj):
+        """write video object by OpenCV"""
+        obj, fps = obj
+        if isinstance(obj, np.ndarray):
+            vr = obj
+            width, height = vr[0].shape[1], vr[0].shape[0]
+            fourcc = cv2.VideoWriter_fourcc(*"mp4v")  # Alternatively, use 'XVID'
+            out = cv2.VideoWriter(out_path, fourcc, fps, (width, height))
+            for frame in vr:
+                out.write(frame)
+            out.release()
+        else:
+            raise TypeError("Unsupported object type")
+
+
 class _BaseJsonWriterBackend(object):
     def __init__(self, indent=4, ensure_ascii=False):
         super().__init__()
@@ -310,7 +381,7 @@ class _BaseJsonWriterBackend(object):
 
 class JsonWriterBackend(_BaseJsonWriterBackend):
     def _write_obj(self, out_path, obj, **bk_args):
-        with open(out_path, "w") as f:
+        with open(out_path, "w", encoding="utf-8") as f:
             json.dump(obj, f, **bk_args)
 
 
@@ -374,3 +445,15 @@ class PandasCSVWriterBackend(_CSVWriterBackend):
         else:
             raise TypeError("Unsupported object type")
         return ts.to_csv(out_path)
+
+
+class MarkdownWriterBackend(_BaseWriterBackend):
+    """MarkdownWriterBackend"""
+
+    def __init__(self):
+        super().__init__()
+
+    def _write_obj(self, out_path, obj):
+        """write markdown obj"""
+        with open(out_path, mode="w", encoding="utf-8", errors="replace") as f:
+            f.write(obj)

paddlex/inference/utils/new_ir_blacklist.py

@@ -18,5 +18,11 @@ NEWIR_BLOCKLIST = [
     "TimesNet_ad",
     "Nonstationary_ad",
     "DLinear_ad",
+    "Co-Deformable-DETR-R50",
+    "Co-Deformable-DETR-Swin-T",
+    "Co-DINO-R50",
+    "Co-DINO-Swin-L",
     "LaTeX_OCR_rec",
+    "BEVFusion",
+    "GroundingDINO-T",
 ]