paddlex 2.0.0rc4__py3-none-any.whl → 3.0.0b2__py3-none-any.whl

paddlex/utils/device.py

@@ -0,0 +1,103 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import GPUtil
+import lazy_paddle as paddle
+
+from . import logging
+from .errors import raise_unsupported_device_error
+
+SUPPORTED_DEVICE_TYPE = ["cpu", "gpu", "xpu", "npu", "mlu"]
+
+
+def _constr_device(device_type, device_ids):
+    if device_ids:
+        device_ids = ",".join(map(str, device_ids))
+        return f"{device_type}:{device_ids}"
+    else:
+        return f"{device_type}"
+
+
+def get_default_device():
+    avail_gpus = GPUtil.getAvailable()
+    if not avail_gpus:
+        return "cpu"
+    else:
+        return _constr_device("gpu", [avail_gpus[0]])
+
+
+def parse_device(device):
+    """parse_device"""
+    # According to https://www.paddlepaddle.org.cn/documentation/docs/zh/api/paddle/device/set_device_cn.html
+    parts = device.split(":")
+    if len(parts) > 2:
+        raise ValueError(f"Invalid device: {device}")
+    if len(parts) == 1:
+        device_type, device_ids = parts[0], None
+    else:
+        device_type, device_ids = parts
+        device_ids = device_ids.split(",")
+        for device_id in device_ids:
+            if not device_id.isdigit():
+                raise ValueError(
+                    f"Device ID must be an integer. Invalid device ID: {device_id}"
+                )
+        device_ids = list(map(int, device_ids))
+    device_type = device_type.lower()
+    # raise_unsupported_device_error(device_type, SUPPORTED_DEVICE_TYPE)
+    assert device_type.lower() in SUPPORTED_DEVICE_TYPE
+    return device_type, device_ids
+
+
+def update_device_num(device, num):
+    device_type, device_ids = parse_device(device)
+    if device_ids:
+        assert len(device_ids) >= num
+        return _constr_device(device_type, device_ids[:num])
+    else:
+        return _constr_device(device_type, device_ids)
+
+
+def set_env_for_device(device):
+    def _set(envs):
+        for key, val in envs.items():
+            os.environ[key] = val
+            logging.debug(f"{key} has been set to {val}.")
+
+    device_type, device_ids = parse_device(device)
+    if device_type.lower() in ["gpu", "xpu", "npu", "mlu"]:
+        if device_type.lower() == "gpu" and paddle.is_compiled_with_rocm():
+            envs = {"FLAGS_conv_workspace_size_limit": "2000"}
+            _set(envs)
+        if device_type.lower() == "npu":
+            envs = {
+                "FLAGS_npu_jit_compile": "0",
+                "FLAGS_use_stride_kernel": "0",
+                "FLAGS_allocator_strategy": "auto_growth",
+                "CUSTOM_DEVICE_BLACK_LIST": "pad3d,pad3d_grad,set_value,set_value_with_tensor",
+                "FLAGS_npu_scale_aclnn": "True",
+                "FLAGS_npu_split_aclnn": "True",
+            }
+            _set(envs)
+        if device_type.lower() == "xpu":
+            envs = {
+                "BKCL_FORCE_SYNC": "1",
+                "BKCL_TIMEOUT": "1800",
+                "FLAGS_use_stride_kernel": "0",
+            }
+            _set(envs)
+        if device_type.lower() == "mlu":
+            envs = {"FLAGS_use_stride_kernel": "0"}
+            _set(envs)

paddlex/utils/download.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,190 +13,176 @@
 # limitations under the License.
 
 import os
-import os.path as osp
-import shutil
-import requests
-import tqdm
+import sys
 import time
-import hashlib
+import shutil
 import tarfile
 import zipfile
-import filelock
-import paddle
-from . import logging
-
-DOWNLOAD_RETRY_LIMIT = 3
-
-
-def md5check(fullname, md5sum=None):
-    if md5sum is None:
-        return True
-
-    logging.info("File {} md5 checking...".format(fullname))
-    md5 = hashlib.md5()
-    with open(fullname, 'rb') as f:
-        for chunk in iter(lambda: f.read(4096), b""):
-            md5.update(chunk)
-    calc_md5sum = md5.hexdigest()
-
-    if calc_md5sum != md5sum:
-        logging.info("File {} md5 check failed, {}(calc) != "
-                     "{}(base)".format(fullname, calc_md5sum, md5sum))
-        return False
-    return True
-
-
-def move_and_merge_tree(src, dst):
-    """
-    Move src directory to dst, if dst is already exists,
-    merge src to dst
-    """
-    if not osp.exists(dst):
-        shutil.move(src, dst)
-    else:
-        for fp in os.listdir(src):
-            src_fp = osp.join(src, fp)
-            dst_fp = osp.join(dst, fp)
-            if osp.isdir(src_fp):
-                if osp.isdir(dst_fp):
-                    move_and_merge_tree(src_fp, dst_fp)
-                else:
-                    shutil.move(src_fp, dst_fp)
-            elif osp.isfile(src_fp) and \
-                    not osp.isfile(dst_fp):
-                shutil.move(src_fp, dst_fp)
-
-
-def download(url, path, md5sum=None):
-    """
-    Download from url, save to path.
-
-    url (str): download url
-    path (str): download to given path
-    """
-    if not osp.exists(path):
-        os.makedirs(path)
-
-    fname = osp.split(url)[-1]
-    fullname = osp.join(path, fname)
-    retry_cnt = 0
-    while not (osp.exists(fullname) and md5check(fullname, md5sum)):
-        if retry_cnt < DOWNLOAD_RETRY_LIMIT:
-            retry_cnt += 1
+import tempfile
+
+import requests
+
+__all__ = ["download", "extract", "download_and_extract"]
+
+
+class _ProgressPrinter(object):
+    """ProgressPrinter"""
+
+    def __init__(self, flush_interval=0.1):
+        super().__init__()
+        self._last_time = 0
+        self._flush_intvl = flush_interval
+
+    def print(self, str_, end=False):
+        """print"""
+        if end:
+            str_ += "\n"
+            self._last_time = 0
+        if time.time() - self._last_time >= self._flush_intvl:
+            sys.stdout.write(f"\r{str_}")
+            self._last_time = time.time()
+            sys.stdout.flush()
+
+
+def _download(url, save_path, print_progress):
+    if print_progress:
+        print(f"Connecting to {url} ...")
+
+    with requests.get(url, stream=True, timeout=15) as r:
+        r.raise_for_status()
+
+        total_length = r.headers.get("content-length")
+
+        if total_length is None:
+            with open(save_path, "wb") as f:
+                shutil.copyfileobj(r.raw, f)
         else:
-            logging.debug("{} download failed.".format(fname))
-            raise RuntimeError("Download from {} failed. "
-                               "Retry limit reached".format(url))
-
-        logging.info("Downloading {} from {}".format(fname, url))
-
-        req = requests.get(url, stream=True)
-        if req.status_code != 200:
-            raise RuntimeError("Downloading from {} failed with code "
-                               "{}!".format(url, req.status_code))
-
-        # For protecting download interupted, download to
-        # tmp_fullname firstly, move tmp_fullname to fullname
-        # after download finished
-        tmp_fullname = fullname + "_tmp"
-        total_size = req.headers.get('content-length')
-        with open(tmp_fullname, 'wb') as f:
-            if total_size:
-                download_size = 0
-                current_time = time.time()
-                for chunk in tqdm.tqdm(
-                        req.iter_content(chunk_size=1024),
-                        total=(int(total_size) + 1023) // 1024,
-                        unit='KB'):
-                    f.write(chunk)
-                    download_size += 1024
-                    if download_size % 524288 == 0:
-                        total_size_m = round(
-                            int(total_size) / 1024.0 / 1024.0, 2)
-                        download_size_m = round(download_size / 1024.0 /
-                                                1024.0, 2)
-                        speed = int(524288 /
-                                    (time.time() - current_time + 0.01) /
-                                    1024.0)
-                        current_time = time.time()
-                        logging.debug(
-                            "Downloading: TotalSize={}M, DownloadSize={}M, Speed={}KB/s"
-                            .format(total_size_m, download_size_m, speed))
-            else:
-                for chunk in req.iter_content(chunk_size=1024):
-                    if chunk:
-                        f.write(chunk)
-        shutil.move(tmp_fullname, fullname)
-        logging.debug("{} download completed.".format(fname))
-
-    return fullname
-
-
-def decompress(fname):
-    """
-    Decompress for zip and tar file
-    """
-    logging.info("Decompressing {}...".format(fname))
-
-    # For protecting decompressing interupted,
-    # decompress to fpath_tmp directory firstly, if decompress
-    # successed, move decompress files to fpath and delete
-    # fpath_tmp and remove download compress file.
-    fpath = osp.split(fname)[0]
-    fpath_tmp = osp.join(fpath, 'tmp')
-    if osp.isdir(fpath_tmp):
-        shutil.rmtree(fpath_tmp)
-        os.makedirs(fpath_tmp)
-
-    if fname.find('tar') >= 0 or fname.find('tgz') >= 0:
-        with tarfile.open(fname) as tf:
-            tf.extractall(path=fpath_tmp)
-    elif fname.find('zip') >= 0:
-        with zipfile.ZipFile(fname) as zf:
-            zf.extractall(path=fpath_tmp)
-    else:
-        raise TypeError("Unsupport compress file type {}".format(fname))
-
-    for f in os.listdir(fpath_tmp):
-        src_dir = osp.join(fpath_tmp, f)
-        dst_dir = osp.join(fpath, f)
-        move_and_merge_tree(src_dir, dst_dir)
-
-    shutil.rmtree(fpath_tmp)
-    logging.debug("{} decompressed.".format(fname))
-    return dst_dir
-
-
-def url2dir(url, path):
-    download(url, path)
-    if url.endswith(('tgz', 'tar.gz', 'tar', 'zip')):
-        fname = osp.split(url)[-1]
-        savepath = osp.join(path, fname)
-        return decompress(savepath)
-
-
-def download_and_decompress(url, path='.'):
-    nranks = paddle.distributed.get_world_size()
-    local_rank = paddle.distributed.get_rank()
-    fname = osp.split(url)[-1]
-    fullname = osp.join(path, fname)
-    # if url.endswith(('tgz', 'tar.gz', 'tar', 'zip')):
-    #     fullname = osp.join(path, fname.split('.')[0])
-    if nranks <= 1:
-        dst_dir = url2dir(url, path)
-        if dst_dir is not None:
-            fullname = dst_dir
+            with open(save_path, "wb") as f:
+                dl = 0
+                total_length = int(total_length)
+                if print_progress:
+                    printer = _ProgressPrinter()
+                    print(f"Downloading {os.path.basename(save_path)} ...")
+                for data in r.iter_content(chunk_size=4096):
+                    dl += len(data)
+                    f.write(data)
+                    if print_progress:
+                        done = int(50 * dl / total_length)
+                        printer.print(
+                            f"[{'=' * done:<50s}] {float(100 * dl) / total_length:.2f}%"
+                        )
+            if print_progress:
+                printer.print(f"[{'=' * 50:<50s}] {100:.2f}%", end=True)
+
+
+def _extract_zip_file(file_path, extd_dir):
+    """extract zip file"""
+    with zipfile.ZipFile(file_path, "r") as f:
+        file_list = f.namelist()
+        total_num = len(file_list)
+        for index, file in enumerate(file_list):
+            f.extract(file, extd_dir)
+            yield total_num, index
+
+
+def _extract_tar_file(file_path, extd_dir):
+    """extract tar file"""
+    try:
+        with tarfile.open(file_path, "r:*") as f:
+            file_list = f.getnames()
+            total_num = len(file_list)
+            for index, file in enumerate(file_list):
+                try:
+                    f.extract(file, extd_dir)
+                except KeyError:
+                    print(f"File {file} not found in the archive.")
+                yield total_num, index
+    except Exception as e:
+        print(f"An error occurred: {e}")
+
+
+def _extract(file_path, extd_dir, print_progress):
+    """extract"""
+    if print_progress:
+        printer = _ProgressPrinter()
+        print(f"Extracting {os.path.basename(file_path)}")
+
+    if zipfile.is_zipfile(file_path):
+        handler = _extract_zip_file
+    elif tarfile.is_tarfile(file_path):
+        handler = _extract_tar_file
     else:
-        lock_path = fullname + '.lock'
-        if not os.path.exists(fullname):
-            with open(lock_path, 'w'):
-                os.utime(lock_path, None)
-            if local_rank == 0:
-                dst_dir = url2dir(url, path)
-                if dst_dir is not None:
-                    fullname = dst_dir
-                os.remove(lock_path)
+        raise RuntimeError("Unsupported file format.")
+
+    for total_num, index in handler(file_path, extd_dir):
+        if print_progress:
+            done = int(50 * float(index) / total_num)
+            printer.print(f"[{'=' * done:<50s}] {float(100 * index) / total_num:.2f}%")
+    if print_progress:
+        printer.print(f"[{'=' * 50:<50s}] {100:.2f}%", end=True)
+
+
+def _remove_if_exists(path):
+    """remove"""
+    if os.path.exists(path):
+        if os.path.isdir(path):
+            shutil.rmtree(path)
+        else:
+            os.remove(path)
+
+
+def download(url, save_path, print_progress=True, overwrite=False):
+    """download"""
+    os.makedirs(os.path.dirname(save_path), exist_ok=True)
+    if overwrite:
+        _remove_if_exists(save_path)
+    if not os.path.exists(save_path):
+        _download(url, save_path, print_progress=print_progress)
+
+
+def extract(file_path, extd_dir, print_progress=True):
+    """extract"""
+    return _extract(file_path, extd_dir, print_progress=print_progress)
+
+
+def download_and_extract(
+    url, save_dir, dst_name, print_progress=True, overwrite=False, no_interm_dir=True
+):
+    """download and extract"""
+    # NOTE: `url` MUST come from a trusted source, since we do not provide a solution
+    # to secure against CVE-2007-4559.
+    os.makedirs(save_dir, exist_ok=True)
+    dst_path = os.path.join(save_dir, dst_name)
+    if overwrite:
+        _remove_if_exists(dst_path)
+
+    if not os.path.exists(dst_path):
+        with tempfile.TemporaryDirectory() as td:
+            arc_file_path = os.path.join(td, url.split("/")[-1])
+            extd_dir = os.path.splitext(arc_file_path)[0]
+            _download(url, arc_file_path, print_progress=print_progress)
+            tmp_extd_dir = os.path.join(td, "extract")
+            _extract(arc_file_path, tmp_extd_dir, print_progress=print_progress)
+            if no_interm_dir:
+                file_names = os.listdir(tmp_extd_dir)
+                if len(file_names) == 1:
+                    file_name = file_names[0]
+                else:
+                    file_name = dst_name
+                sp = os.path.join(tmp_extd_dir, file_name)
+                if not os.path.exists(sp):
+                    raise FileNotFoundError
+                dp = os.path.join(save_dir, file_name)
+                if os.path.isdir(sp):
+                    shutil.copytree(sp, dp, symlinks=True)
+                else:
+                    shutil.copyfile(sp, dp)
+                extd_file = dp
             else:
-                while os.path.exists(lock_path):
-                    time.sleep(1)
-    return fullname
+                shutil.copytree(tmp_extd_dir, extd_dir)
+                extd_file = extd_dir
+
+            if not os.path.exists(dst_path) or not os.path.samefile(
+                extd_file, dst_path
+            ):
+                shutil.move(extd_file, dst_path)

paddlex/utils/errors/__init__.py

@@ -0,0 +1,17 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from .dataset_checker import *
+from .others import *

paddlex/utils/errors/dataset_checker.py

@@ -0,0 +1,78 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+__all__ = [
+    "FailedError",
+    "CheckFailedError",
+    "ConvertFailedError",
+    "SplitFailedError",
+    "AnalyseFailedError",
+    "CheckFailedError",
+    "DatasetFileNotFoundError",
+]
+
+
+class FailedError(Exception):
+    """base error class"""
+
+    def __init__(self, err_info=None, solution=None, message=None):
+        if message is None:
+            message = self._construct_message(err_info, solution)
+        super().__init__(message)
+
+    def _construct_message(self, err_info, solution):
+        if err_info is None:
+            return ""
+        else:
+            msg = (
+                f"{self.mode} failed. We encountered the following error:\n  {err_info}"
+            )
+            if solution is not None:
+                msg += f"\nPlease try to resolve the issue as follows:\n  {solution}"
+            return msg
+
+
+class CheckFailedError(FailedError):
+    """check dataset error"""
+
+    mode = "Check dataset"
+
+
+class ConvertFailedError(FailedError):
+    """convert dataset error"""
+
+    mode = "Convert dataset"
+
+
+class SplitFailedError(FailedError):
+    """split dataset error"""
+
+    mode = "Split dataset"
+
+
+class AnalyseFailedError(FailedError):
+    """analyse dataset error"""
+
+    mode = "Analyse dataset"
+
+
+class DatasetFileNotFoundError(CheckFailedError):
+    """dataset file not found error"""
+
+    def __init__(self, file_path=None, err_info=None, solution=None, message=None):
+        if err_info is None:
+            if file_path is not None:
+                err_info = f"{file_path} does not exist."
+        super().__init__(err_info, solution, message)