paddlex 3.0.0b2__py3-none-any.whl → 3.0.0rc0__py3-none-any.whl

paddlex/inference/models/base/predictor/base_predictor.py

@@ -0,0 +1,175 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List, Dict, Any, Iterator
+from pathlib import Path
+from abc import abstractmethod, ABC
+
+from ....utils.io import YAMLReader
+from ....common.batch_sampler import BaseBatchSampler
+
+
+class PredictionWrap:
+    """Wraps the prediction data and supports get by index."""
+
+    def __init__(self, data: Dict[str, List[Any]], num: int) -> None:
+        """Initializes the PredictionWrap with prediction data.
+
+        Args:
+            data (Dict[str, List[Any]]): A dictionary where keys are string identifiers and values are lists of predictions.
+            num (int): The number of predictions, that is length of values per key in the data dictionary.
+
+        Raises:
+            AssertionError: If the length of any list in data does not match num.
+        """
+        assert isinstance(data, dict), "data must be a dictionary"
+        for k in data:
+            assert len(data[k]) == num, f"{len(data[k])} != {num} for key {k}!"
+        self._data = data
+        self._keys = data.keys()
+
+    def get_by_idx(self, idx: int) -> Dict[str, Any]:
+        """Get the prediction by specified index.
+
+        Args:
+            idx (int): The index to get predictions from.
+
+        Returns:
+            Dict[str, Any]: A dictionary with the same keys as the input data, but with the values at the specified index.
+        """
+        return {key: self._data[key][idx] for key in self._keys}
+
+
+class BasePredictor(ABC):
+    """BasePredictor."""
+
+    MODEL_FILE_PREFIX = "inference"
+
+    def __init__(self, model_dir: str, config: Dict = None) -> None:
+        """Initializes the BasePredictor.
+
+        Args:
+            model_dir (str): The directory where the static model files is stored.
+            config (dict, optional): The configuration of model to infer. Defaults to None.
+        """
+        super().__init__()
+        self.model_dir = Path(model_dir)
+        self.config = config if config else self.load_config(self.model_dir)
+        self.batch_sampler = self._build_batch_sampler()
+        self.result_class = self._get_result_class()
+
+        # alias predict() to the __call__()
+        self.predict = self.__call__
+        self.benchmark = None
+
+    @property
+    def config_path(self) -> str:
+        """
+        Get the path to the configuration file.
+
+        Returns:
+            str: The path to the configuration file.
+        """
+        return self.get_config_path(self.model_dir)
+
+    @property
+    def model_name(self) -> str:
+        """
+        Get the model name.
+
+        Returns:
+            str: The model name.
+        """
+        return self.config["Global"]["model_name"]
+
+    @classmethod
+    def get_config_path(cls, model_dir) -> str:
+        """Get the path to the configuration file for the given model directory.
+
+        Args:
+            model_dir (Path): The directory where the static model files is stored.
+
+        Returns:
+            Path: The path to the configuration file.
+        """
+        return model_dir / f"{cls.MODEL_FILE_PREFIX}.yml"
+
+    @classmethod
+    def load_config(cls, model_dir) -> Dict:
+        """Load the configuration from the specified model directory.
+
+        Args:
+            model_dir (Path): The where the static model files is stored.
+
+        Returns:
+            dict: The loaded configuration dictionary.
+        """
+        yaml_reader = YAMLReader()
+        return yaml_reader.read(cls.get_config_path(model_dir))
+
+    @abstractmethod
+    def __call__(self, input: Any, **kwargs: Dict[str, Any]) -> Iterator[Any]:
+        """Predict with the given input and additional keyword arguments."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def set_predictor(self, batch_size: int = None, device: str = None, *args) -> None:
+        """Sets up the predictor."""
+        raise NotImplementedError
+
+    def apply(self, input: Any, **kwargs) -> Iterator[Any]:
+        """
+        Do predicting with the input data and yields predictions.
+
+        Args:
+            input (Any): The input data to be predicted.
+
+        Yields:
+            Iterator[Any]: An iterator yielding prediction results.
+        """
+        for batch_data in self.batch_sampler(input):
+            prediction = self.process(batch_data, **kwargs)
+            prediction = PredictionWrap(prediction, len(batch_data))
+            for idx in range(len(batch_data)):
+                yield self.result_class(prediction.get_by_idx(idx))
+
+    @abstractmethod
+    def process(self, batch_data: List[Any]) -> Dict[str, List[Any]]:
+        """process the batch data sampled from BatchSampler and return the prediction result.
+
+        Args:
+            batch_data (List[Any]): The batch data sampled from BatchSampler.
+
+        Returns:
+            Dict[str, List[Any]]: The prediction result.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def _build_batch_sampler(self) -> BaseBatchSampler:
+        """Build batch sampler.
+
+        Returns:
+            BaseBatchSampler: batch sampler object.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def _get_result_class(self) -> type:
+        """Get the result class.
+
+        Returns:
+            type: The result class.
+        """
+        raise NotImplementedError

paddlex/inference/models/base/predictor/basic_predictor.py

@@ -0,0 +1,139 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Dict, Any, Iterator
+from abc import abstractmethod
+
+from .....utils.subclass_register import AutoRegisterABCMetaClass
+from .....utils.flags import (
+    INFER_BENCHMARK,
+    INFER_BENCHMARK_WARMUP,
+)
+from .....utils import logging
+from ....utils.pp_option import PaddlePredictorOption
+from ....utils.benchmark import benchmark
+from .base_predictor import BasePredictor
+
+
+class BasicPredictor(
+    BasePredictor,
+    metaclass=AutoRegisterABCMetaClass,
+):
+    """BasicPredictor."""
+
+    __is_base = True
+
+    def __init__(
+        self,
+        model_dir: str,
+        config: Dict[str, Any] = None,
+        device: str = None,
+        batch_size: int = 1,
+        pp_option: PaddlePredictorOption = None,
+    ) -> None:
+        """Initializes the BasicPredictor.
+
+        Args:
+            model_dir (str): The directory where the model files are stored.
+            config (Dict[str, Any], optional): The configuration dictionary. Defaults to None.
+            device (str, optional): The device to run the inference engine on. Defaults to None.
+            batch_size (int, optional): The batch size to predict. Defaults to 1.
+            pp_option (PaddlePredictorOption, optional): The inference engine options. Defaults to None.
+        """
+        super().__init__(model_dir=model_dir, config=config)
+        if not pp_option:
+            pp_option = PaddlePredictorOption(model_name=self.model_name)
+        if device:
+            pp_option.device = device
+        trt_dynamic_shapes = (
+            self.config.get("Hpi", {})
+            .get("backend_configs", {})
+            .get("paddle_infer", {})
+            .get("trt_dynamic_shapes", None)
+        )
+        if trt_dynamic_shapes:
+            pp_option.trt_dynamic_shapes = trt_dynamic_shapes
+        self.pp_option = pp_option
+        self.pp_option.batch_size = batch_size
+        self.batch_sampler.batch_size = batch_size
+
+        logging.debug(f"{self.__class__.__name__}: {self.model_dir}")
+        self.benchmark = benchmark
+
+    def __call__(
+        self,
+        input: Any,
+        batch_size: int = None,
+        device: str = None,
+        pp_option: PaddlePredictorOption = None,
+        **kwargs: Dict[str, Any],
+    ) -> Iterator[Any]:
+        """
+        Predict with the input data.
+
+        Args:
+            input (Any): The input data to be predicted.
+            batch_size (int, optional): The batch size to use. Defaults to None.
+            device (str, optional): The device to run the predictor on. Defaults to None.
+            pp_option (PaddlePredictorOption, optional): The predictor options to set. Defaults to None.
+            **kwargs (Dict[str, Any]): Additional keyword arguments to set up predictor.
+
+        Returns:
+            Iterator[Any]: An iterator yielding the prediction output.
+        """
+        self.set_predictor(batch_size, device, pp_option)
+        if self.benchmark:
+            self.benchmark.start()
+            if INFER_BENCHMARK_WARMUP > 0:
+                output = self.apply(input, **kwargs)
+                warmup_num = 0
+                for _ in range(INFER_BENCHMARK_WARMUP):
+                    try:
+                        next(output)
+                        warmup_num += 1
+                    except StopIteration:
+                        logging.warning(
+                            f"There are only {warmup_num} batches in input data, but `INFER_BENCHMARK_WARMUP` has been set to {INFER_BENCHMARK_WARMUP}."
+                        )
+                        break
+                self.benchmark.warmup_stop(warmup_num)
+            output = list(self.apply(input, **kwargs))
+            self.benchmark.collect(len(output))
+        else:
+            yield from self.apply(input, **kwargs)
+
+    def set_predictor(
+        self,
+        batch_size: int = None,
+        device: str = None,
+        pp_option: PaddlePredictorOption = None,
+    ) -> None:
+        """
+        Sets the predictor configuration.
+
+        Args:
+            batch_size (int, optional): The batch size to use. Defaults to None.
+            device (str, optional): The device to run the predictor on. Defaults to None.
+            pp_option (PaddlePredictorOption, optional): The predictor options to set. Defaults to None.
+
+        Returns:
+            None
+        """
+        if batch_size:
+            self.batch_sampler.batch_size = batch_size
+            self.pp_option.batch_size = batch_size
+        if device and device != self.pp_option.device:
+            self.pp_option.device = device
+        if pp_option and pp_option != self.pp_option:
+            self.pp_option = pp_option

paddlex/inference/models/common/__init__.py

@@ -0,0 +1,35 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .vision import (
+    F,
+    Resize,
+    ResizeByLong,
+    ResizeByShort,
+    Normalize,
+    ToCHWImage,
+    ToBatch,
+)
+
+from .ts import (
+    BuildTSDataset,
+    TSCutOff,
+    TSNormalize,
+    TimeFeature,
+    TStoArray,
+    TStoBatch,
+)
+from .static_infer import StaticInfer
+
+from .tokenizer import PretrainedTokenizer, PretrainedTokenizer

paddlex/inference/models/common/static_infer.py

@@ -0,0 +1,329 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Union, Tuple, List, Dict, Any, Iterator
+import os
+import shutil
+from pathlib import Path
+import lazy_paddle as paddle
+import numpy as np
+
+from ....utils.flags import DEBUG, FLAGS_json_format_model, USE_PIR_TRT
+from ....utils import logging
+from ...utils.pp_option import PaddlePredictorOption
+
+
+def collect_trt_shapes(
+    model_file, model_params, gpu_id, shape_range_info_path, trt_dynamic_shapes
+):
+    config = paddle.inference.Config(model_file, model_params)
+    config.enable_use_gpu(100, gpu_id)
+    min_arrs, opt_arrs, max_arrs = {}, {}, {}
+    for name, candidate_shapes in trt_dynamic_shapes.items():
+        min_shape, opt_shape, max_shape = candidate_shapes
+        min_arrs[name] = np.ones(min_shape, dtype=np.float32)
+        opt_arrs[name] = np.ones(opt_shape, dtype=np.float32)
+        max_arrs[name] = np.ones(max_shape, dtype=np.float32)
+
+    config.collect_shape_range_info(shape_range_info_path)
+    predictor = paddle.inference.create_predictor(config)
+    # opt_arrs would be used twice to simulate the most common situations
+    for arrs in [min_arrs, opt_arrs, opt_arrs, max_arrs]:
+        for name, arr in arrs.items():
+            input_handler = predictor.get_input_handle(name)
+            input_handler.reshape(arr.shape)
+            input_handler.copy_from_cpu(arr)
+        predictor.run()
+
+
+def convert_trt(mode, pp_model_path, trt_save_path, trt_dynamic_shapes):
+    from lazy_paddle.tensorrt.export import (
+        Input,
+        TensorRTConfig,
+        convert,
+        PrecisionMode,
+    )
+
+    precision_map = {
+        "trt_int8": PrecisionMode.INT8,
+        "trt_fp32": PrecisionMode.FP32,
+        "trt_fp16": PrecisionMode.FP16,
+    }
+    trt_inputs = []
+    for name, candidate_shapes in trt_dynamic_shapes.items():
+        min_shape, opt_shape, max_shape = candidate_shapes
+        trt_input = Input(
+            min_input_shape=min_shape,
+            optim_input_shape=opt_shape,
+            max_input_shape=max_shape,
+        )
+        trt_inputs.append(trt_input)
+
+    # Create TensorRTConfig
+    trt_config = TensorRTConfig(inputs=trt_inputs)
+    trt_config.precision_mode = precision_map[mode]
+    trt_config.save_model_dir = trt_save_path
+    convert(pp_model_path, trt_config)
+
+
+class Copy2GPU:
+
+    def __init__(self, input_handlers):
+        super().__init__()
+        self.input_handlers = input_handlers
+
+    def __call__(self, x):
+        for idx in range(len(x)):
+            self.input_handlers[idx].reshape(x[idx].shape)
+            self.input_handlers[idx].copy_from_cpu(x[idx])
+
+
+class Copy2CPU:
+
+    def __init__(self, output_handlers):
+        super().__init__()
+        self.output_handlers = output_handlers
+
+    def __call__(self):
+        output = []
+        for out_tensor in self.output_handlers:
+            batch = out_tensor.copy_to_cpu()
+            output.append(batch)
+        return output
+
+
+class Infer:
+
+    def __init__(self, predictor):
+        super().__init__()
+        self.predictor = predictor
+
+    def __call__(self):
+        self.predictor.run()
+
+
+class StaticInfer:
+    """Predictor based on Paddle Inference"""
+
+    def __init__(
+        self, model_dir: str, model_prefix: str, option: PaddlePredictorOption
+    ) -> None:
+        super().__init__()
+        self.model_dir = model_dir
+        self.model_prefix = model_prefix
+        self._update_option(option)
+
+    def _update_option(self, option: PaddlePredictorOption) -> None:
+        if self.option and option == self.option:
+            return
+        self._option = option
+        self._reset()
+
+    @property
+    def option(self) -> PaddlePredictorOption:
+        return self._option if hasattr(self, "_option") else None
+
+    @option.setter
+    def option(self, option: Union[None, PaddlePredictorOption]) -> None:
+        if option:
+            self._update_option(option)
+
+    def _reset(self) -> None:
+        logging.debug(f"Env: {self.option}")
+        (
+            predictor,
+            input_handlers,
+            output_handlers,
+        ) = self._create()
+        self.copy2gpu = Copy2GPU(input_handlers)
+        self.copy2cpu = Copy2CPU(output_handlers)
+        self.infer = Infer(predictor)
+        self.option.changed = False
+
+    def _create(
+        self,
+    ) -> Tuple[
+        "paddle.base.libpaddle.PaddleInferPredictor",
+        "paddle.base.libpaddle.PaddleInferTensor",
+        "paddle.base.libpaddle.PaddleInferTensor",
+    ]:
+        """_create"""
+        from lazy_paddle.inference import Config, create_predictor
+
+        if FLAGS_json_format_model:
+            model_file = (self.model_dir / f"{self.model_prefix}.json").as_posix()
+        # when FLAGS_json_format_model is not set, use inference.json if exist, otherwise inference.pdmodel
+        else:
+            model_file = self.model_dir / f"{self.model_prefix}.json"
+            if model_file.exists():
+                model_file = model_file.as_posix()
+            # default by `pdmodel` suffix
+            else:
+                model_file = (
+                    self.model_dir / f"{self.model_prefix}.pdmodel"
+                ).as_posix()
+        params_file = (self.model_dir / f"{self.model_prefix}.pdiparams").as_posix()
+
+        # for TRT
+        if self.option.run_mode.startswith("trt"):
+            assert self.option.device == "gpu"
+            if not USE_PIR_TRT:
+                if self.option.shape_info_filename is None:
+                    shape_range_info_path = (
+                        self.model_dir / "shape_range_info.pbtxt"
+                    ).as_posix()
+                else:
+                    shape_range_info_path = self.option.shape_info_filename
+                if not os.path.exists(shape_range_info_path):
+                    logging.info(
+                        f"Dynamic shape info is collected into: {shape_range_info_path}"
+                    )
+                    collect_trt_shapes(
+                        model_file,
+                        params_file,
+                        self.option.device_id,
+                        shape_range_info_path,
+                        self.option.trt_dynamic_shapes,
+                    )
+                else:
+                    logging.info(
+                        f"A dynamic shape info file ( {shape_range_info_path} ) already exists. No need to collect again."
+                    )
+                self.option.shape_info_filename = shape_range_info_path
+            else:
+                trt_save_path = (
+                    Path(self.model_dir) / "trt" / self.model_prefix
+                ).as_posix()
+                pp_model_path = (Path(self.model_dir) / self.model_prefix).as_posix()
+                convert_trt(
+                    self.option.run_mode,
+                    pp_model_path,
+                    trt_save_path,
+                    self.option.trt_dynamic_shapes,
+                )
+                model_file = trt_save_path + ".json"
+                params_file = trt_save_path + ".pdiparams"
+
+        config = Config(model_file, params_file)
+        if self.option.device == "gpu":
+            config.exp_disable_mixed_precision_ops({"feed", "fetch"})
+            config.enable_use_gpu(100, self.option.device_id)
+            if not self.option.run_mode.startswith("trt"):
+                if hasattr(config, "enable_new_ir"):
+                    config.enable_new_ir(self.option.enable_new_ir)
+                if hasattr(config, "enable_new_executor"):
+                    config.enable_new_executor()
+                config.set_optimization_level(3)
+            # NOTE: The pptrt settings are not aligned with those of FD.
+            else:
+                if not USE_PIR_TRT:
+                    precision_map = {
+                        "trt_int8": Config.Precision.Int8,
+                        "trt_fp32": Config.Precision.Float32,
+                        "trt_fp16": Config.Precision.Half,
+                    }
+                    config.enable_tensorrt_engine(
+                        workspace_size=(1 << 30) * self.option.batch_size,
+                        max_batch_size=self.option.batch_size,
+                        min_subgraph_size=self.option.min_subgraph_size,
+                        precision_mode=precision_map[self.option.run_mode],
+                        use_static=self.option.trt_use_static,
+                        use_calib_mode=self.option.trt_calib_mode,
+                    )
+                    config.enable_tuned_tensorrt_dynamic_shape(
+                        self.option.shape_info_filename, True
+                    )
+        elif self.option.device == "npu":
+            config.enable_custom_device("npu")
+            if hasattr(config, "enable_new_executor"):
+                config.enable_new_executor()
+        elif self.option.device == "xpu":
+            if hasattr(config, "enable_new_executor"):
+                config.enable_new_executor()
+        elif self.option.device == "mlu":
+            config.enable_custom_device("mlu")
+            if hasattr(config, "enable_new_executor"):
+                config.enable_new_executor()
+        elif self.option.device == "dcu":
+            config.enable_use_gpu(100, self.option.device_id)
+            if hasattr(config, "enable_new_executor"):
+                config.enable_new_executor()
+            # XXX: is_compiled_with_rocm() must be True on dcu platform ?
+            if paddle.is_compiled_with_rocm():
+                # Delete unsupported passes in dcu
+                config.delete_pass("conv2d_add_act_fuse_pass")
+                config.delete_pass("conv2d_add_fuse_pass")
+        else:
+            assert self.option.device == "cpu"
+            config.disable_gpu()
+            if "mkldnn" in self.option.run_mode:
+                try:
+                    config.enable_mkldnn()
+                    if "bf16" in self.option.run_mode:
+                        config.enable_mkldnn_bfloat16()
+                except Exception as e:
+                    logging.warning(
+                        "MKL-DNN is not available. We will disable MKL-DNN."
+                    )
+                config.set_mkldnn_cache_capacity(-1)
+            else:
+                if hasattr(config, "disable_mkldnn"):
+                    config.disable_mkldnn()
+            config.set_cpu_math_library_num_threads(self.option.cpu_threads)
+
+            if hasattr(config, "enable_new_ir"):
+                config.enable_new_ir(self.option.enable_new_ir)
+            if hasattr(config, "enable_new_executor"):
+                config.enable_new_executor()
+            config.set_optimization_level(3)
+
+        config.enable_memory_optim()
+        for del_p in self.option.delete_pass:
+            config.delete_pass(del_p)
+
+        # Disable paddle inference logging
+        if not DEBUG:
+            config.disable_glog_info()
+
+        predictor = create_predictor(config)
+
+        # Get input and output handlers
+        input_names = predictor.get_input_names()
+        input_names.sort()
+        input_handlers = []
+        output_handlers = []
+        for input_name in input_names:
+            input_handler = predictor.get_input_handle(input_name)
+            input_handlers.append(input_handler)
+        output_names = predictor.get_output_names()
+        for output_name in output_names:
+            output_handler = predictor.get_output_handle(output_name)
+            output_handlers.append(output_handler)
+        return predictor, input_handlers, output_handlers
+
+    def __call__(self, x) -> List[Any]:
+        if self.option.changed:
+            self._reset()
+        self.copy2gpu(x)
+        self.infer()
+        pred = self.copy2cpu()
+        return pred
+
+    @property
+    def benchmark(self):
+        return {
+            "Copy2GPU": self.copy2gpu,
+            "Infer": self.infer,
+            "Copy2CPU": self.copy2cpu,
+        }

paddlex/inference/models/common/tokenizer/__init__.py

@@ -0,0 +1,17 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .tokenizer_utils import PretrainedTokenizer
+from .gpt_tokenizer import GPTTokenizer
+from .bert_tokenizer import BertTokenizer