paddlex 3.0.0b2__py3-none-any.whl → 3.0.0rc1__py3-none-any.whl

paddlex/repo_manager/utils.py

@@ -1,4 +1,4 @@
-# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,15 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import contextlib
+import importlib.metadata
 import os
-import sys
-import json
 import platform
 import subprocess
-import contextlib
-
-from parsley import makeGrammar
+import sys
 
+from ..utils import logging
+from ..utils.env import get_device_type
 
 PLATFORM = platform.system()
 
@@ -29,52 +29,72 @@ def _check_call(*args, **kwargs):
     return subprocess.check_call(*args, **kwargs)
 
 
-def _check_output(*args, **kwargs):
-    return subprocess.check_output(*args, **kwargs)
-
-
-def check_installation_using_pip(pkg):
-    """check_installation_using_pip"""
-    out = _check_output(["pip", "list", "--format", "json"])
-    out = out.rstrip()
-    lst = json.loads(out)
-    return any(ele["name"] == pkg for ele in lst)
-
-
-def uninstall_package_using_pip(pkg):
-    """uninstall_package_using_pip"""
-    return _check_call([sys.executable, "-m", "pip", "uninstall", "-y", pkg])
-
-
-def install_packages_using_pip(
-    pkgs, editable=False, req_files=None, cons_files=None, no_deps=False, pip_flags=None
-):
-    """install_packages_using_pip"""
-    args = [sys.executable, "-m", "pip", "install"]
-    if editable:
-        args.append("-e")
-    if req_files is not None:
-        for req_file in req_files:
-            args.append("-r")
-            args.append(req_file)
-    if cons_files is not None:
-        for cons_file in cons_files:
-            args.append("-c")
-            args.append(cons_file)
-    if isinstance(pkgs, str):
-        pkgs = [pkgs]
-    args.extend(pkgs)
-    if pip_flags is not None:
-        args.extend(pip_flags)
-    return _check_call(args)
-
-
-def install_deps_using_pip():
-    """install requirements"""
-    current_file_path = os.path.dirname(os.path.abspath(__file__))
-    deps_path = os.path.join(current_file_path, "requirements.txt")
-    args = [sys.executable, "-m", "pip", "install", "-r", deps_path]
-    return _check_call(args)
+def _compare_version(version1, version2):
+    import re
+
+    def parse_version(version_str):
+        version_pattern = re.compile(
+            r"^(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?:-(?P<pre_release>.*))?(?:\+(?P<build_metadata>.+))?$"
+        )
+        match = version_pattern.match(version_str)
+        if not match:
+            raise ValueError(f"Unexpected version string: {version_str}")
+        return (
+            int(match.group("major")),
+            int(match.group("minor")),
+            int(match.group("patch")),
+            match.group("pre_release"),
+        )
+
+    v1_infos = parse_version(version1)
+    v2_infos = parse_version(version2)
+    for v1_info, v2_info in zip(v1_infos, v2_infos):
+        if v1_info is None and v2_info is None:
+            continue
+        if v1_info is None or (v2_info is not None and v1_info < v2_info):
+            return -1
+        if v2_info is None or (v1_info is not None and v1_info > v2_info):
+            return 1
+    return 0
+
+
+def check_package_installation(package):
+    try:
+        importlib.metadata.distribution(package)
+    except importlib.metadata.PackageNotFoundError:
+        return False
+    return True
+
+
+def install_external_deps(repo_name, repo_root):
+    """install paddle repository custom dependencies"""
+    import paddle
+
+    def get_gcc_version():
+        return subprocess.check_output(["gcc", "--version"]).decode("utf-8").split()[2]
+
+    if repo_name == "PaddleDetection":
+        if os.path.exists(os.path.join(repo_root, "ppdet", "ext_op")):
+            """Install custom op for rotated object detection"""
+            if (
+                PLATFORM == "Linux"
+                and _compare_version(get_gcc_version(), "8.2.0") >= 0
+                and "gpu" in get_device_type()
+                and (
+                    paddle.is_compiled_with_cuda()
+                    and not paddle.is_compiled_with_rocm()
+                )
+            ):
+                with switch_working_dir(os.path.join(repo_root, "ppdet", "ext_op")):
+                    # TODO: Apply constraints here
+                    args = [sys.executable, "setup.py", "install"]
+                    _check_call(args)
+            else:
+                logging.warning(
+                    "The custom operators in PaddleDetection for Rotated Object Detection is only supported when using CUDA, GCC>=8.2.0 and Paddle>=2.0.1, "
+                    "your environment does not meet these requirements, so we will skip the installation of custom operators under PaddleDetection/ppdet/ext_ops, "
+                    "which means you can not train the Rotated Object Detection models."
+                )
 
 
 def clone_repo_using_git(url, branch=None):
@@ -109,18 +129,6 @@ def remove_repo_using_rm(name):
             return _check_call(["rm", "-rf", name])
 
 
-def build_wheel_using_pip(pkg, dst_dir="./", with_deps=False, pip_flags=None):
-    """build_wheel_using_pip"""
-    args = [sys.executable, "-m", "pip", "wheel", "--wheel-dir", dst_dir]
-    if not with_deps:
-        args.append("--no-deps")
-    if pip_flags is not None:
-        args.extend(pip_flags)
-    args.append(pkg)
-
-    return _check_call(args)
-
-
 @contextlib.contextmanager
 def mute():
     """mute"""
@@ -138,161 +146,3 @@ def switch_working_dir(new_wd):
         yield
     finally:
         os.chdir(cwd)
-
-
-def _build_dep_spec_pep508_grammar():
-    # Refer to https://peps.python.org/pep-0508/
-    grammar = """
-        wsp           = ' ' | '\t'
-        version_cmp   = wsp* <'<=' | '<' | '!=' | '==' | '>=' | '>' | '~=' | '==='>
-        version       = wsp* <(letterOrDigit | '-' | '_' | '.' | '*' | '+' | '!')+>
-        version_one   = version_cmp:op version:v wsp* -> (op, v)
-        version_many  = version_one:v1 (wsp* ',' version_one)*:v2 -> [v1] + v2
-        versionspec   = ('(' version_many:v ')' ->v) | version_many
-        urlspec       = '@' wsp* <uri_reference>
-        marker_op     = version_cmp | (wsp* 'in') | (wsp* 'not' wsp+ 'in')
-        python_str_c  = (wsp | letter | digit | '(' | ')' | '.' | '{' | '}' |
-                        '-' | '_' | '*' | '#' | ':' | ';' | ',' | '/' | '?' |
-                        '[' | ']' | '!' | '~' | '`' | '@' | '$' | '%' | '^' |
-                        '&' | '=' | '+' | '|' | '<' | '>' )
-        dquote        = '"'
-        squote        = '\\''
-        comment       = '#' <anything*>:s end -> s
-        python_str    = (squote <(python_str_c | dquote)*>:s squote |
-                        dquote <(python_str_c | squote)*>:s dquote) -> s
-        env_var       = ('python_version' | 'python_full_version' |
-                        'os_name' | 'sys_platform' | 'platform_release' |
-                        'platform_system' | 'platform_version' |
-                        'platform_machine' | 'platform_python_implementation' |
-                        'implementation_name' | 'implementation_version' |
-                        'extra' # ONLY when defined by a containing layer
-                        )
-        marker_var    = wsp* (env_var | python_str)
-        marker_expr   = marker_var:l marker_op:o marker_var:r -> (o, l, r)
-                    | wsp* '(' marker:m wsp* ')' -> m
-        marker_and    = marker_expr:l wsp* 'and' marker_expr:r -> ('and', l, r)
-                    | marker_expr:m -> m
-        marker_or     = marker_and:l wsp* 'or' marker_and:r -> ('or', l, r)
-                        | marker_and:m -> m
-        marker        = marker_or
-        quoted_marker = ';' wsp* marker
-        identifier_end = letterOrDigit | (('-' | '_' | '.' )* letterOrDigit)
-        identifier    = <letterOrDigit identifier_end* >
-        name          = identifier
-        extras_list   = identifier:i (wsp* ',' wsp* identifier)*:ids -> [i] + ids
-        extras        = '[' wsp* extras_list?:e wsp* ']' -> e
-        name_req      = (name:n wsp* extras?:e wsp* versionspec?:v wsp* quoted_marker?:m
-                        -> (n, e or [], v or [], m))
-        url_req       = (name:n wsp* extras?:e wsp* urlspec:v (wsp+ | end) quoted_marker?:m
-                        -> (n, e or [], v, m))
-        specification = wsp* (url_req | name_req):s wsp* comment? -> s
-        # The result is a tuple - name, list-of-extras,
-        # list-of-version-constraints-or-a-url, marker-ast or None
-
-
-        uri_reference = <uri | relative_ref>
-        uri           = scheme ':' hier_part ('?' query )? ('#' fragment)?
-        hier_part     = ('//' authority path_abempty) | path_absolute | path_rootless | path_empty
-        absolute_uri  = scheme ':' hier_part ('?' query )?
-        relative_ref  = relative_part ('?' query )? ('#' fragment )?
-        relative_part = '//' authority path_abempty | path_absolute | path_noscheme | path_empty
-        scheme        = letter (letter | digit | '+' | '-' | '.')*
-        authority     = (userinfo '@' )? host (':' port )?
-        userinfo      = (unreserved | pct_encoded | sub_delims | ':')*
-        host          = ip_literal | ipv4_address | reg_name
-        port          = digit*
-        ip_literal    = '[' (ipv6_address | ipvfuture) ']'
-        ipvfuture     = 'v' hexdig+ '.' (unreserved | sub_delims | ':')+
-        ipv6_address   = (
-                        (h16 ':'){6} ls32
-                        | '::' (h16 ':'){5} ls32
-                        | (h16 )?  '::' (h16 ':'){4} ls32
-                        | ((h16 ':')? h16 )? '::' (h16 ':'){3} ls32
-                        | ((h16 ':'){0,2} h16 )? '::' (h16 ':'){2} ls32
-                        | ((h16 ':'){0,3} h16 )? '::' h16 ':' ls32
-                        | ((h16 ':'){0,4} h16 )? '::' ls32
-                        | ((h16 ':'){0,5} h16 )? '::' h16
-                        | ((h16 ':'){0,6} h16 )? '::' )
-        h16           = hexdig{1,4}
-        ls32          = (h16 ':' h16) | ipv4_address
-        ipv4_address   = dec_octet '.' dec_octet '.' dec_octet '.' dec_octet
-        nz            = ~'0' digit
-        dec_octet     = (
-                        digit # 0-9
-                        | nz digit # 10-99
-                        | '1' digit{2} # 100-199
-                        | '2' ('0' | '1' | '2' | '3' | '4') digit # 200-249
-                        | '25' ('0' | '1' | '2' | '3' | '4' | '5') )# %250-255
-        reg_name = (unreserved | pct_encoded | sub_delims)*
-        path = (
-                path_abempty # begins with '/' or is empty
-                | path_absolute # begins with '/' but not '//'
-                | path_noscheme # begins with a non-colon segment
-                | path_rootless # begins with a segment
-                | path_empty ) # zero characters
-        path_abempty  = ('/' segment)*
-        path_absolute = '/' (segment_nz ('/' segment)* )?
-        path_noscheme = segment_nz_nc ('/' segment)*
-        path_rootless = segment_nz ('/' segment)*
-        path_empty    = pchar{0}
-        segment       = pchar*
-        segment_nz    = pchar+
-        segment_nz_nc = (unreserved | pct_encoded | sub_delims | '@')+
-                        # non-zero-length segment without any colon ':'
-        pchar         = unreserved | pct_encoded | sub_delims | ':' | '@'
-        query         = (pchar | '/' | '?')*
-        fragment      = (pchar | '/' | '?')*
-        pct_encoded   = '%' hexdig
-        unreserved    = letter | digit | '-' | '.' | '_' | '~'
-        reserved      = gen_delims | sub_delims
-        gen_delims    = ':' | '/' | '?' | '#' | '(' | ')?' | '@'
-        sub_delims    = '!' | '$' | '&' | '\\'' | '(' | ')' | '*' | '+' | ',' | ';' | '='
-        hexdig        = digit | 'a' | 'A' | 'b' | 'B' | 'c' | 'C' | 'd' | 'D' | 'e' | 'E' | 'f' | 'F'
-    """
-
-    compiled = makeGrammar(grammar, {})
-    return compiled
-
-
-_pep508_grammar = None
-
-
-def to_dep_spec_pep508(s):
-    """to_dep_spec_pep508"""
-    global _pep508_grammar
-    if _pep508_grammar is None:
-        _pep508_grammar = _build_dep_spec_pep508_grammar()
-    parsed = _pep508_grammar(s)
-    return parsed.specification()
-
-
-def env_marker_ast2expr(marker_ast):
-    """env_marker_ast2expr"""
-    MARKER_VARS = (
-        "python_version",
-        "python_full_version",
-        "os_name",
-        "sys_platform",
-        "platform_release",
-        "platform_system",
-        "platform_version",
-        "platform_machine",
-        "platform_python_implementation",
-        "implementation_name",
-        "implementation_version",
-        "extra",  # ONLY when defined by a containing layer
-    )
-    o, l, r = marker_ast
-    if isinstance(l, tuple):
-        l = env_marker_ast2expr(l)
-    else:
-        assert isinstance(l, str)
-        if l not in MARKER_VARS:
-            l = repr(l)
-    if isinstance(r, tuple):
-        r = env_marker_ast2expr(r)
-    else:
-        assert isinstance(r, str)
-        if r not in MARKER_VARS:
-            r = repr(r)
-    return f"{l} {o} {r}"

paddlex/utils/__init__.py

@@ -1,4 +1,4 @@
-# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

paddlex/utils/cache.py

@@ -1,4 +1,4 @@
-# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,19 +13,17 @@
 # limitations under the License.
 
 
+import atexit
+import functools
+import hashlib
+import inspect
 import os
 import os.path as osp
-from pathlib import Path
-import inspect
-import functools
 import pickle
-import hashlib
 import tempfile
-import atexit
-import filelock
-
-from . import logging
+from pathlib import Path
 
+import filelock
 
 DEFAULT_CACHE_DIR = osp.abspath(osp.join(os.path.expanduser("~"), ".paddlex"))
 CACHE_DIR = os.environ.get("PADDLE_PDX_CACHE_HOME", DEFAULT_CACHE_DIR)
@@ -123,7 +121,7 @@ class TempFileManager:
             try:
                 temp_file.close()
                 os.remove(temp_file.name)
-            except FileNotFoundError as e:
+            except FileNotFoundError:
                 pass
         self.temp_files = []
 

paddlex/utils/config.py

@@ -1,4 +1,4 @@
-# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,12 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
-import copy
 import argparse
+import copy
+import os
+
 import yaml
+
 from . import logging
-from .errors import raise_key_not_found_error
+from .file_interface import custom_open
 
 __all__ = ["get_config"]
 
@@ -29,7 +31,7 @@ class AttrDict(dict):
         if key in self:
             return self[key]
         else:
-            raise raise_key_not_found_error(key, self)
+            raise AttributeError(key)
 
     def __setattr__(self, key, value):
         if key in self.__dict__:
@@ -61,7 +63,7 @@ def create_attr_dict(yaml_config):
 
 def parse_config(cfg_file):
     """Load a config file into AttrDict"""
-    with open(cfg_file, "r") as fopen:
+    with custom_open(cfg_file, "r") as fopen:
         yaml_config = AttrDict(yaml.load(fopen, Loader=yaml.SafeLoader))
     create_attr_dict(yaml_config)
     return yaml_config
@@ -70,7 +72,7 @@ def parse_config(cfg_file):
 def print_dict(d, delimiter=0):
     """
     Recursively visualize a dict and
-    indenting acrrording by the relationship of keys.
+    indenting according by the relationship of keys.
     """
     placeholder = "-" * 60
     for k, v in sorted(d.items()):
@@ -124,7 +126,7 @@ def override(dl, ks, v):
             return s
 
     assert isinstance(dl, (list, dict)), "{} should be a list or a dict"
-    assert len(ks) > 0, "lenght of keys should larger than 0"
+    assert len(ks) > 0, "length of keys should larger than 0"
     if isinstance(dl, list):
         k = parse_str(ks[0])
         if len(ks) == 1: