paddlex 2.0.0rc4__py3-none-any.whl → 3.0.0b2__py3-none-any.whl

paddlex/repo_manager/core.py

@@ -0,0 +1,271 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+from collections import OrderedDict
+
+from ..utils import logging
+from .utils import install_deps_using_pip
+from .meta import get_all_repo_names, get_repo_meta
+from .repo import (
+    build_repo_instance,
+    build_repo_group_getter,
+    build_repo_group_installer,
+)
+
+__all__ = [
+    "set_parent_dirs",
+    "setup",
+    "wheel",
+    "is_initialized",
+    "initialize",
+    "get_versions",
+]
+
+
+def _parse_repo_deps(repos):
+    ret = []
+    for repo_name in repos:
+        repo_meta = get_repo_meta(repo_name)
+        ret.extend(_parse_repo_deps(repo_meta.get("requires", [])))
+        ret.append(repo_name)
+    return ret
+
+
+class _GlobalContext(object):
+    REPO_PARENT_DIR = None
+    PDX_COLLECTION_MOD = None
+    REPOS = None
+
+    @classmethod
+    def set_parent_dirs(cls, repo_parent_dir, pdx_collection_mod):
+        """set_parent_dirs"""
+        cls.REPO_PARENT_DIR = repo_parent_dir
+        cls.PDX_COLLECTION_MOD = pdx_collection_mod
+
+    @classmethod
+    def build_repo_instance(cls, repo_name):
+        """build_repo_instance"""
+        return build_repo_instance(
+            repo_name, cls.REPO_PARENT_DIR, cls.PDX_COLLECTION_MOD
+        )
+
+    @classmethod
+    def is_initialized(cls):
+        """is_initialized"""
+        return cls.REPOS is not None
+
+    @classmethod
+    def initialize(cls):
+        """initialize"""
+        cls.REPOS = []
+
+    @classmethod
+    def add_repo(cls, repo):
+        """add_repo"""
+        if not cls.is_initialized():
+            cls.initialize()
+        cls.REPOS.append(repo)
+
+    @classmethod
+    def add_repos(cls, repos):
+        """add_repos"""
+        if len(repos) == 0 and not cls.is_initialized():
+            cls.initialize()
+        for repo in repos:
+            cls.add_repo(repo)
+
+
+set_parent_dirs = _GlobalContext.set_parent_dirs
+is_initialized = _GlobalContext.is_initialized
+
+
+def setup(
+    repo_names,
+    no_deps=False,
+    constraints=None,
+    platform=None,
+    update_repos=False,
+    use_local_repos=False,
+):
+    """setup"""
+    if update_repos and use_local_repos:
+        logging.error(
+            f"The `--update_repos` and `--use_local_repos` should not be True at the same time. They are global setting for all repos. `--update_repos` means that update all repos to sync with remote, and `--use_local_repos` means that don't update when local repo is exsting."
+        )
+        raise Exception()
+
+    repo_names = list(set(_parse_repo_deps(repo_names)))
+
+    repos = []
+    for repo_name in repo_names:
+        repo = _GlobalContext.build_repo_instance(repo_name)
+        repos.append(repo)
+
+    changed_repos = []
+    repos_to_get = []
+    for repo in repos:
+        repo_name = repo.name
+        if repo.check_repo_exiting():
+            if use_local_repos:
+                # when use_local_repos has been set, it can be only assume that the local repo has changed, otherwise there is no need to specify.
+                changed_repos.append(repo_name)
+                logging.warning(
+                    f"We will use the existing repo of {repo.name} and the repo will be reinstall."
+                )
+                continue
+
+            logging.warning(f"Existing of {repo.name} repo.")
+            if update_repos:
+                remove_existing = True
+            else:
+                if sys.stdin.isatty():
+                    logging.warning("Should we remove it (y/n)?")
+                try:
+                    remove_existing = input()
+                except EOFError:
+                    logging.warning(
+                        "Unable to read from stdin. Please set `--use_local_repos` to \
+                        True or False to apply a global setting for using exsting or re-getting repos."
+                    )
+                    raise
+                remove_existing = remove_existing.lower() in ("y", "yes")
+
+            if remove_existing:
+                changed_repos.append(repo_name)
+                repo.remove()
+                logging.warning(f"Existing {repo.name} repo has been removed.")
+                repos_to_get.append(repo)
+            else:
+                logging.warning(f"We will use the existing repo of {repo.name}.")
+        else:
+            changed_repos.append(repo)
+            repos_to_get.append(repo)
+
+    repos_to_install = []
+    for repo in repos:
+        repo_name = repo.name
+        if repo.check_installation():
+            logging.warning(f"Existing installation of {repo.name} detected.")
+            reinstall = repo_name in changed_repos
+            if reinstall:
+                uninstall_existing = True
+            else:
+                if sys.stdin.isatty():
+                    logging.warning("Should we uninstall it (y/n)?")
+                try:
+                    uninstall_existing = input()
+                except EOFError:
+                    logging.warning(
+                        "Unable to read from stdin. Please set `reinstall` to \
+                        True or False to apply a global setting for reinstalling repos."
+                    )
+                    raise
+                uninstall_existing = uninstall_existing.lower() in ("y", "yes")
+
+            if uninstall_existing:
+                repo.uninstall()
+                repos_to_install.append(repo)
+            else:
+                logging.warning(
+                    f"We will use the existing installation of {repo.name}."
+                )
+        else:
+            repos_to_install.append(repo)
+    getter = build_repo_group_getter(*repos_to_get)
+    installer = build_repo_group_installer(*repos_to_install)
+
+    if len(repos_to_get) > 0:
+        logging.info(
+            f"Now download and update the repos: {list(repo.name for repo in repos_to_get)}."
+        )
+        getter.get(force=True, platform=platform)
+        logging.info("All repos are existing.")
+    else:
+        logging.info("No repo need to download or update.")
+
+    if not no_deps:
+        logging.info("Dependencies are listed below:")
+        logging.info(installer.get_deps())
+
+    logging.info("Now installing the packages...")
+    installer.install(force_reinstall=False, no_deps=no_deps, constraints=constraints)
+    install_deps_using_pip()
+    logging.info("All packages are installed.")
+
+
+def wheel(repo_names, dst_dir="./", fail_fast=False):
+    """wheel"""
+    for repo_name in repo_names:
+        repo = _GlobalContext.build_repo_instance(repo_name)
+        logging.info(f"Now building Wheel for {repo_name}...")
+        try:
+            tgt_dir = os.path.join(dst_dir, repo.pkg_name)
+            if os.path.exists(tgt_dir):
+                raise FileExistsError(f"{tgt_dir} already exists.")
+            repo.wheel(tgt_dir)
+        except Exception as e:
+            logging.warning(
+                f"Failed to build wheel for {repo_name}. We encountered the following error:\n  {str(e)}\n"
+            )
+            if fail_fast:
+                raise
+        else:
+            logging.info(f"Wheel for {repo_name} is built.\n")
+
+
+def initialize(repo_names=None):
+    """initialize"""
+    if _GlobalContext.is_initialized():
+        raise RuntimeError(
+            "PDX has already been initialized. Reinitialization is not supported."
+        )
+
+    if repo_names is None:
+        try_all = True
+        repo_names = get_all_repo_names()
+    else:
+        try_all = False
+
+    repos = []
+    for repo_name in repo_names:
+        logging.debug(f"Now initializing {repo_name}...")
+        repo = _GlobalContext.build_repo_instance(repo_name)
+        flag = repo.initialize()
+        if flag:
+            logging.debug(f"{repo_name} is initialized.")
+            repos.append(repo)
+        else:
+            if try_all:
+                logging.debug(
+                    f"Failed to initialize {repo_name}. Please make sure {repo_name} is properly installed."
+                )
+            else:
+                pass
+
+    _GlobalContext.add_repos(repos)
+
+
+def get_versions(repo_names=None):
+    """get_versions"""
+    if repo_names is None:
+        repo_names = get_all_repo_names()
+
+    name2versions = OrderedDict()
+    for repo_name in repo_names:
+        repo = _GlobalContext.build_repo_instance(repo_name)
+        versions = repo.get_version()
+        name2versions[repo_name] = versions
+    return name2versions

paddlex/repo_manager/meta.py

@@ -0,0 +1,143 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__all__ = ["get_all_repo_names"]
+
+REPO_DOWNLOAD_BASE = (
+    "https://paddle-model-ecology.bj.bcebos.com/paddlex/PaddleX3.0/repos/"
+)
+
+REPO_NAMES = [
+    "PaddleClas",
+    "PaddleOCR",
+    "PaddleDetection",
+    "PaddleSeg",
+    "PaddleNLP",
+    "PaddleTS",
+]
+
+REPO_META = {
+    "PaddleSeg": {
+        "git_path": "/PaddlePaddle/PaddleSeg.git",
+        "platform": "github",
+        "branch": "release/2.10",
+        "pkg_name": "paddleseg",
+        "lib_name": "paddleseg",
+        "pdx_pkg_name": "PaddleSeg_api",
+        "editable": False,
+        "extra_req_files": ["Matting/requirements.txt"],
+        "path_env": "PADDLE_PDX_PADDLESEG_PATH",
+    },
+    "PaddleClas": {
+        "git_path": "/PaddlePaddle/PaddleClas.git",
+        "platform": "github",
+        "branch": "release/2.6",
+        "pkg_name": "paddleclas",
+        "lib_name": "paddleclas",
+        "pdx_pkg_name": "PaddleClas_api",
+        # PaddleClas must be installed in non-editable mode, otherwise it throws
+        # an Import error.
+        "editable": False,
+        "path_env": "PADDLE_PDX_PADDLECLAS_PATH",
+    },
+    "PaddleDetection": {
+        "git_path": "/PaddlePaddle/PaddleDetection.git",
+        "platform": "github",
+        "branch": "release/2.8",
+        "pkg_name": "paddledet",
+        "lib_name": "ppdet",
+        "pdx_pkg_name": "PaddleDetection_api",
+        "editable": False,
+        "path_env": "PADDLE_PDX_PADDLEDETECTION_PATH",
+    },
+    "PaddleOCR": {
+        "git_path": "/PaddlePaddle/PaddleOCR.git",
+        "platform": "github",
+        "branch": "release/2.9",
+        "pkg_name": "paddleocr",
+        "lib_name": "paddleocr",
+        "pdx_pkg_name": "PaddleOCR_api",
+        "editable": False,
+        "extra_req_files": [
+            "ppstructure/kie/requirements.txt",
+            "docs/algorithm/formula_recognition/requirements.txt",
+        ],
+        "path_env": "PADDLE_PDX_PADDLEOCR_PATH",
+        "requires": ["PaddleNLP"],
+    },
+    "PaddleTS": {
+        "git_path": "/PaddlePaddle/PaddleTS.git",
+        "platform": "github",
+        "branch": "release_v1.1",
+        "pkg_name": "paddlets",
+        "lib_name": "paddlets",
+        "pdx_pkg_name": "PaddleTS_api",
+        "editable": False,
+        "path_env": "PADDLE_PDX_PADDLETS_PATH",
+        "pdx_pkg_deps": ["pandas", "ruamel.yaml"],
+    },
+    "PaddleNLP": {
+        "git_path": "/PaddlePaddle/PaddleNLP.git",
+        "platform": "github",
+        "branch": "release/2.9",
+        "pkg_name": "paddlenlp",
+        "lib_name": "paddlenlp",
+        "pdx_pkg_name": "PaddleNLP_api",
+        "editable": False,
+        "path_env": "PADDLE_PDX_PADDLENLP_PATH",
+    },
+    "PaddleSpeech": {
+        "git_path": "/PaddlePaddle/PaddleSpeech.git",
+        "platform": "github",
+        "branch": "develop",
+        "pkg_name": "paddlespeech",
+        "lib_name": "paddlespeech",
+        "pdx_pkg_name": "PaddleSpeech_api",
+        "editable": False,
+        "path_env": "PADDLE_PDX_PADDLESPEECH_PATH",
+        "requires": ["PaddleNLP"],
+    },
+    "PARL": {
+        "git_path": "/PaddlePaddle/PARL.git",
+        "platform": "github",
+        "branch": "develop",
+        "pkg_name": "parl",
+        "lib_name": "parl",
+        "pdx_pkg_name": "PARL_api",
+        "editable": False,
+        "path_env": "PADDLE_PDX_PARL_PATH",
+    },
+    "PaddleMIX": {
+        "git_path": "/PaddlePaddle/PaddleMIX.git",
+        "platform": "github",
+        "branch": "develop",
+        "pkg_name": "paddlemix",
+        "lib_name": "paddlemix",
+        "pdx_pkg_name": "PaddleMIX_api",
+        "editable": True,
+        "extra_editable": "ppdiffusers",
+        "path_env": "PADDLE_PDX_PADDLEMIX_PATH",
+        "requires": ["PaddleNLP"],
+    },
+}
+
+
+def get_repo_meta(repo_name):
+    """get_repo_meta"""
+    return REPO_META[repo_name]
+
+
+def get_all_repo_names():
+    """get_all_repo_names"""
+    return REPO_NAMES