py2docfx 0.1.9.dev1917798__py3-none-any.whl → 0.1.9.dev1926139__py3-none-any.whl

py2docfx/__main__.py

@@ -1,11 +1,13 @@
 from __future__ import annotations # Avoid A | B annotation break under <= py3.9
 import asyncio
 import argparse
+import logging
 import os
 import sys
 import shutil
 
 from py2docfx import PACKAGE_ROOT
+from py2docfx.docfx_yaml.logger import get_logger
 from py2docfx.convert_prepare.constants import SOURCE_REPO, TARGET_REPO, DIST_TEMP
 from py2docfx.convert_prepare.generate_document import generate_document
 from py2docfx.convert_prepare.get_source import YAML_OUTPUT_ROOT
@@ -14,11 +16,6 @@ from py2docfx.convert_prepare.params import load_file_params, load_command_param
 from py2docfx.convert_prepare.package_info import PackageInfo
 import py2docfx.convert_prepare.environment as py2docfxEnvironment
 
-
-print("Adding yaml extension to path")
-sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)),'docfx_yaml'))
-os.chdir(PACKAGE_ROOT)
-
 def get_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(
         description=(
@@ -144,26 +141,40 @@ def get_parser() -> argparse.ArgumentParser:
         help="""A list containing relative paths to the root of the package of files/directories
                         excluded when generating documents, should follow fnmatch-style.""",
     )
+    
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        help="""Increase output verbosity. Cannot be used together with --show-warning""",
+    )
+    
+    parser.add_argument(
+        "--show-warning",
+        action="store_true",
+        help="""Show warning message. Cannot be used together with --verbose""",
+    )
     return parser
 
 
 def parse_command_line_args(argv) -> (
-        list[PackageInfo], list[PackageInfo], str, str, str | os.PathLike, bool):
+        list[PackageInfo], list[PackageInfo], str, str, str | os.PathLike, bool, bool):
     parser = get_parser()
     args = parser.parse_args(argv)
 
     github_token = args.github_token
     ado_token = args.ado_token
     output_root = args.output_root
+    verbose = args.verbose
+    show_warning = args.show_warning
 
     if args.param_file_path:
         (package_info_list, required_packages) = load_file_params(args.param_file_path)
         return (list(package_info_list), list(required_packages), github_token,
-                ado_token, output_root)
+                ado_token, output_root, verbose, show_warning)
     elif args.param_json:
         (package_info_list, required_packages) = load_command_params(args.param_json)
         return (package_info_list, required_packages, github_token,
-                ado_token, output_root)
+                ado_token, output_root, verbose, show_warning)
     else:
         package_info = PackageInfo()
         if not args.install_type:
@@ -194,11 +205,11 @@ def parse_command_line_args(argv) -> (
             package_info.folder = args.folder
             if not package_info.url:
                 if not package_info.folder:
-                    raise ValueError(
-                        "When install_type is source_code, folder or url should be provided"
-                    )
+                    msg = "When install_type is source_code, folder or url should be provided"
+                    raise ValueError(msg)
                 else:
-                    print(f'Read source code from local folder: {package_info.folder}')
+                    msg = f"Read source code from local folder: {package_info.folder}"
+                    logging.info(msg)
 
         if package_info.install_type == PackageInfo.InstallType.DIST_FILE:
             package_info.location = args.location
@@ -208,7 +219,7 @@ def parse_command_line_args(argv) -> (
                     "None",
                     condition="When install_type is dist_file",
                 )
-        return ([package_info], [], github_token, ado_token, output_root)
+        return ([package_info], [], github_token, ado_token, output_root, verbose, show_warning)
 
 async def donwload_package_generate_documents(
         package_info_list: list[PackageInfo],
@@ -216,6 +227,8 @@ async def donwload_package_generate_documents(
         output_doc_folder: os.PathLike | None,
         github_token: str, ado_token: str, required_package_list: list):
     
+    docfx_logger = get_logger(__name__)
+    
     start_num = len(required_package_list)
     env_prepare_tasks = []
     env_remove_tasks = []
@@ -229,12 +242,16 @@ async def donwload_package_generate_documents(
             py2docfxEnvironment.prepare_base_venv(required_package_list, github_token, ado_token))
 
     for idx, package in enumerate(package_info_list):
+        os.environ['PROCESSING_PACKAGE_NAME'] = package.name
         package_number = start_num + idx
-        print(f"Processing package {package.name}, env_prepare_tasks: {len(env_prepare_tasks)}")
+        msg = f"Processing package {package.name}, env_prepare_tasks: {len(env_prepare_tasks)}"
+        docfx_logger.info(msg)
+
         try:
             await env_prepare_tasks[idx]
         except Exception as e:
-            print(f"Failed to setup venv for package {package.name}: {e}")
+            msg = f"Failed to setup venv for package {package.name}: {e}"
+            docfx_logger.error(msg)
             raise
 
         generate_document(package, output_root,
@@ -249,7 +266,10 @@ async def donwload_package_generate_documents(
 
         if idx + py2docfxEnvironment.VENV_BUFFER < len(package_info_list):
             buffer_package_idx = idx + py2docfxEnvironment.VENV_BUFFER
-            print(f"Creating venv {buffer_package_idx}")
+            
+            msg = f"Creating venv {buffer_package_idx}"
+            docfx_logger.info(msg)
+            
             env_prepare_tasks.append(
                 asyncio.create_task(py2docfxEnvironment.prepare_venv(buffer_package_idx, 
                                                                      package_info_list[buffer_package_idx], 
@@ -262,7 +282,8 @@ async def donwload_package_generate_documents(
                 py2docfxEnvironment.remove_environment(idx-1)))
 
         if idx > py2docfxEnvironment.VENV_BUFFER and env_remove_tasks[idx-py2docfxEnvironment.VENV_BUFFER] != None:
-            print(f"Removing venv {idx-py2docfxEnvironment.VENV_BUFFER}")
+            msg = f"Removing venv {idx-py2docfxEnvironment.VENV_BUFFER}"
+            docfx_logger.info(msg)
             await env_remove_tasks[idx-py2docfxEnvironment.VENV_BUFFER]
 
     if output_doc_folder:
@@ -295,24 +316,58 @@ def temp_folder_clean_up(folder_list: list[str | os.PathLike]) -> None:
         if os.path.exists(folder):
             shutil.rmtree(folder)
 
-def main(argv) -> int:
-    clean_up_folder_list = [py2docfxEnvironment.VENV_DIR, DIST_TEMP, SOURCE_REPO, TARGET_REPO]
-    temp_folder_clean_up(clean_up_folder_list)
+def decide_global_log_level(verbose: bool, show_warning: bool) -> None:
+    if verbose and show_warning:
+        raise ValueError("Cannot use --verbose and --show-warning at the same time")
+    if verbose:
+        os.environ['LOG_LEVEL'] = 'INFO'
+        return
+    if show_warning:
+        os.environ['LOG_LEVEL'] = 'WARNING'
+        return
+    
+    # Default log level
+    os.environ['LOG_LEVEL'] = 'ERROR'
 
+def prepare_log_folder() -> None:
+    if not os.path.exists('logs'):
+        os.makedirs('logs')
+        os.makedirs(os.path.join('logs', 'package_logs'))
+    else:
+        shutil.rmtree('logs')
+        os.makedirs('logs')
+        os.makedirs(os.path.join('logs', 'package_logs'))
+
+def main(argv) -> int:
     # TODO: may need to purge pip cache
     (package_info_list,
      required_package_list,
      github_token, ado_token,
-     output_root) = parse_command_line_args(argv)
+     output_root, verbose,
+     show_warning) = parse_command_line_args(argv)
     
+    decide_global_log_level(verbose, show_warning)
+    prepare_log_folder()
+    
+    docfx_logger = get_logger(__name__)
+
+    msg = "Adding yaml extension to path"
+    docfx_logger.info(msg)
+    
+    sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)),'docfx_yaml'))
+    os.chdir(PACKAGE_ROOT)
     output_doc_folder = prepare_out_dir(output_root)
+    
+    clean_up_folder_list = [py2docfxEnvironment.VENV_DIR, DIST_TEMP, SOURCE_REPO, TARGET_REPO]
+    temp_folder_clean_up(clean_up_folder_list)
 
     try:
         asyncio.run(donwload_package_generate_documents(
             package_info_list, output_root, output_doc_folder,
             github_token, ado_token, required_package_list))
     except Exception as e:
-        print(f"An error occurred: {e}")
+        msg = f"An error occurred: {e}"
+        docfx_logger.error(msg)
         raise
     return 0
 

py2docfx/convert_prepare/environment.py

@@ -6,6 +6,7 @@ from py2docfx import PACKAGE_ROOT
 from py2docfx.convert_prepare.package_info import PackageInfo
 from py2docfx.convert_prepare.get_source import get_source
 from py2docfx.convert_prepare import pip_utils
+from py2docfx.docfx_yaml.logger import get_logger
 
 REQUIREMENT_MODULES = ["setuptools", "sphinx==6.1.3", "pyyaml", "jinja2==3.0.3", "wheel"]
 VENV_REQUIREMENT_MODULES = ["setuptools", "wheel"] # to support running setup.py in venv
@@ -27,7 +28,7 @@ def install_converter_requirements(executable: str):
     https://apidrop.visualstudio.com/Content%20CI/_git/ReferenceAutomation?path=/Python/InstallPackage.ps1&line=15&lineEnd=35&lineStartColumn=1&lineEndColumn=87&lineStyle=plain&_a=contents
     """
     pip_install_cmd = [executable, "-m", "pip", "install", "--upgrade"]
-
+    docfx_logger = get_logger(__name__)
     pip_install_common_options = [
         "--no-cache-dir",
         "--quiet",
@@ -37,7 +38,8 @@ def install_converter_requirements(executable: str):
     ]
 
     for module in REQUIREMENT_MODULES:
-        print(f"<CI INFO>: Upgrading {module}...")
+        msg = f"<CI INFO>: Upgrading {module}..."
+        docfx_logger.info(msg)
         subprocess.run(
             pip_install_cmd + [module] + pip_install_common_options, check=True
         )
@@ -45,7 +47,8 @@ def install_converter_requirements(executable: str):
 async def install_venv_requirements(venv_num: int):
     venv_exe = get_venv_exe(venv_num)
     pip_cmd = ["-m", "pip", "install", "--upgrade"]+ VENV_REQUIREMENT_MODULES
-    await(await asyncio.create_subprocess_exec(venv_exe, *pip_cmd)).wait()
+    # await(await asyncio.create_subprocess_exec(venv_exe, *pip_cmd)).wait()
+    await pip_utils.run_async_subprocess(venv_exe, pip_cmd)
 
 def get_venv_path(venv_num: int) -> str:
     return os.path.join(PACKAGE_ROOT, VENV_DIR, "venv"+str(venv_num))
@@ -82,7 +85,8 @@ def get_base_venv_sphinx_build_path() -> str:
 
 async def install_converter_requirement_async(executable: str):
     pip_cmd = PIP_INSTALL_COMMAND + PIP_INSTALL_VENV_COMMON_OPTIONS + REQUIREMENT_MODULES
-    await(await asyncio.create_subprocess_exec(executable, *pip_cmd)).wait()
+    # await(await asyncio.create_subprocess_exec(executable, *pip_cmd)).wait()
+    await pip_utils.run_async_subprocess(executable, pip_cmd)
 
 async def install_required_packages(
         executable: str, required_package_list: list[PackageInfo], github_token: str, ado_token: str):
@@ -92,7 +96,8 @@ async def install_required_packages(
         #     get_source(package, idx, vststoken=ado_token, githubtoken=github_token)
         package_name, options = package.get_install_command()
         pip_cmd = PIP_INSTALL_COMMAND + PIP_INSTALL_VENV_COMMON_OPTIONS + options + [package_name]
-        await(await asyncio.create_subprocess_exec(executable, *pip_cmd)).wait()
+        # await(await asyncio.create_subprocess_exec(executable, *pip_cmd)).wait()
+        await pip_utils.run_async_subprocess(executable, pip_cmd)
 
 async def create_environment(venv_path: int):
     if os.name == 'nt':
@@ -101,26 +106,39 @@ async def create_environment(venv_path: int):
         await (await asyncio.create_subprocess_exec("python3", "-m", "venv", venv_path)).wait()
 
 async def prepare_base_venv(required_package_list: list[PackageInfo], github_token: str, ado_token: str):
-    print(f"<CI INFO>: Creating basevenv...")
+    docfx_logger = get_logger(__name__)
+    
+    msg = f"<CI INFO>: Creating basevenv..."
+    docfx_logger.info(msg)
     await create_environment(get_base_venv_path())
-    print(f"<CI INFO>: Installing converter requirements in ...")
+    
+    msg = f"<CI INFO>: Installing converter requirements in ..."
+    docfx_logger.info(msg)
     await install_converter_requirement_async(get_base_venv_exe())
-    print(f"<CI INFO>: Installing required packages in basevenv...")
+    
+    msg = f"<CI INFO>: Installing required packages in basevenv..."
+    docfx_logger.info(msg)
     await install_required_packages(get_base_venv_exe(), required_package_list, github_token, ado_token)
-    print(f"<CI INFO>: basevenv setup complete.")
+    
+    msg = f"<CI INFO>: basevenv setup complete."
+    docfx_logger.info(msg)
 
 async def prepare_venv(venv_num: int, package_info: PackageInfo, package_number: int, github_token: str, ado_token: str):
+    docfx_logger = get_logger(__name__)
     await create_environment(get_venv_path(venv_num))
     await install_venv_requirements(venv_num)
     get_source(get_venv_exe(venv_num), package_info, package_number, vststoken=ado_token, githubtoken=github_token)
     package_name, options = package_info.get_install_command()
     await pip_utils.install_in_exe_async(get_venv_exe(venv_num), package_name, options)
-    print(f"<CI INFO>: venv{venv_num} setup complete.")
+    msg = f"<CI INFO>: venv{venv_num} setup complete."
+    docfx_logger.info(msg)
 
 async def remove_environment(venv_num: int):
+    docfx_logger = get_logger(__name__)
     venv_path = get_venv_path(venv_num)
     if os.path.exists(venv_path):
-        print(f"<CI INFO>: Removing venv{venv_num}...")
+        msg = f"<CI INFO>: Removing venv{venv_num}..."
+        docfx_logger.info(msg)
         # Create a subprocess to run the shell command for removing the directory
         process = await asyncio.create_subprocess_shell(
             f'rm -rf {venv_path}' if os.name != 'nt' else f'rmdir /S /Q {venv_path}',
@@ -129,6 +147,9 @@ async def remove_environment(venv_num: int):
         )
         stdout, stderr = await process.communicate()
         if process.returncode == 0:
-            print(f"<CI INFO>: venv{venv_num} removed.")
+            msg = f"<CI INFO>: venv{venv_num} removed."
+            docfx_logger.info(msg)
         else:
-            print(f"<CI ERROR>: Failed to remove venv{venv_num}. Error: {stderr.decode()}")
+            msg = f"<CI ERROR>: Failed to remove venv{venv_num}. Error: {stderr.decode()}"
+            docfx_logger.error(msg)
+            raise RuntimeError()

py2docfx/convert_prepare/generate_document.py

@@ -1,7 +1,7 @@
 from __future__ import annotations # Avoid A | B annotation break under <= py3.9
 import os
 import sys
-
+from py2docfx.docfx_yaml.logger import get_logger
 from py2docfx.convert_prepare.generate_conf import generate_conf
 from py2docfx.convert_prepare.git import checkout
 from py2docfx.convert_prepare.package_info import PackageInfo
@@ -11,6 +11,7 @@ from py2docfx.convert_prepare.subpackage import merge_subpackage_files
 CONF_TEMPLATE_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "conf_templates")
 
 def generate_document(pkg: PackageInfo, output_root: str | os.PathLike, sphinx_build_path: str, extra_package_path: str, executable=sys.executable):
+    docfx_logger = get_logger(__name__)
     # Copy manual written RST from target doc repo
     package_paths = pkg.path
     if output_root:
@@ -29,14 +30,20 @@ def generate_document(pkg: PackageInfo, output_root: str | os.PathLike, sphinx_b
         if pkg.install_type == pkg.InstallType.SOURCE_CODE and getattr(pkg, "branch", None):
             checkout(package_paths.source_folder, pkg.branch)
 
-        print(f"<CI INFO>: Generating RST files for {pkg.name}.")
+        msg = f"<CI INFO>: Generating RST files for {pkg.name}."
+        docfx_logger.info(msg)
+
         subpackages_rst_record = run_apidoc(package_paths.doc_folder, package_paths.source_folder,
                                             exclude_paths, pkg)
 
-    print("<CI INFO>: Listing RST files:")
+    msg = f"<CI INFO>: Listing RST files:"
+    docfx_logger.info(msg)
     for rst_file in os.listdir(package_paths.doc_folder):
-        print(rst_file)
-    print("<CI INFO>: Running Sphinx build...")
+        docfx_logger.info(rst_file)
+        
+    msg = "<CI INFO>: Running Sphinx build..."
+    docfx_logger.info(msg)
+
     generate_conf(pkg, package_paths.doc_folder, CONF_TEMPLATE_DIR)
     run_converter(package_paths.doc_folder, package_paths.yaml_output_folder, sphinx_build_path, extra_package_path, executable=executable)
     

py2docfx/convert_prepare/get_source.py

@@ -6,6 +6,7 @@ import sys
 import py2docfx.convert_prepare.git as git
 import py2docfx.convert_prepare.pip_utils as pip_utils
 import py2docfx.convert_prepare.pack as pack
+from py2docfx.docfx_yaml.logger import get_logger
 from py2docfx.convert_prepare.constants import TARGET_REPO, SOURCE_REPO, DIST_TEMP
 from py2docfx.convert_prepare.package_info import PackageInfo
 from py2docfx.convert_prepare.source import Source
@@ -72,6 +73,7 @@ def update_package_info(executable: str, pkg: PackageInfo, source_folder: str):
     )
 
 def get_source(executable: str, pkg: PackageInfo, cnt: int, vststoken=None, githubtoken=None):
+    docfx_logger = get_logger(__name__)
     path_cnt = str(cnt)
     dist_dir = path.join(DIST_TEMP, path_cnt)
 
@@ -120,6 +122,8 @@ def get_source(executable: str, pkg: PackageInfo, cnt: int, vststoken=None, gith
             os.listdir(dist_dir)[0]
         )
     else:
-        raise ValueError(f"Unknown install type: {pkg.install_type}")
+        msg = f"Unknown install type: {pkg.install_type}"
+        docfx_logger.error(msg)
+        raise ValueError()
 
     update_package_info(executable, pkg, source_folder)

py2docfx/convert_prepare/git.py

@@ -1,8 +1,9 @@
 import re
 import subprocess
 
-repoMap = {}
+from py2docfx.docfx_yaml.logger import get_logger
 
+repoMap = {}
 
 def clone(repo_location, branch, folder, extra_token=None):
     """
@@ -12,15 +13,16 @@ def clone(repo_location, branch, folder, extra_token=None):
     replacing
     https://apidrop.visualstudio.com/Content%20CI/_git/ReferenceAutomation?path=/Python/GetPackageCode.ps1&line=50&lineEnd=90&lineStartColumn=1&lineEndColumn=2&lineStyle=plain&_a=contents
     """
-
-    print("<CI INFO>: Cloning repo: {}...".format(repo_location))
+    docfx_logger = get_logger(__name__)
+    msg = "<CI INFO>: Cloning repo: {}...".format(repo_location)
+    docfx_logger.info(msg)
 
     global repoMap
 
     if not test_url(repo_location, extra_token):
-        raise ValueError(
-            "Git repo address {} is not a valid URL.".format(repo_location)
-        )
+        msg = "Git repo address {} is not a valid URL.".format(repo_location)
+        docfx_logger.error(msg)
+        raise ValueError()
     else:
         # Remove http(s):// from url to record. Further avoid dup-clone.
         pureURL = re.sub("^\s*https?://", "", repo_location)
@@ -57,17 +59,17 @@ def clone(repo_location, branch, folder, extra_token=None):
                     ]
                 subprocess.run(clone_params, check=True)
                 repoMap[pureURL] = folder
-                print(
-                    "<CI INFO>: Repo {} successfully cloned in {}...".format(
+                msg = "<CI INFO>: Repo {} successfully cloned in {}...".format(
                         repo_location, repoMap[pureURL]
                     )
-                )
+                docfx_logger.info(msg)
             else:
                 raise ValueError(
                     "Branch {} doesn't exist in repo {}.".format(branch, repo_location)
                 )
         else:
-            print("<CI INFO>: Repo already cloned in {}...".format(repoMap[pureURL]))
+            msg = "<CI INFO>: Repo already cloned in {}...".format(repoMap[pureURL])
+            docfx_logger.info(msg)
 
         return repoMap[pureURL]
 
@@ -76,9 +78,12 @@ def test_url(url, extraHeader):
     """
     Test if the url is a valid git repo url
     """
+    docfx_logger = get_logger(__name__)
     params = ["git", "ls-remote", url]
     if extraHeader:
-        print("Using extra header to test git repo url.")
+        msg = "Using extra header to test git repo url."
+        docfx_logger.info(msg)
+
         params[1:1] = [
             "-c",
             "http.extraHeader=Authorization: Basic {}".format(extraHeader),
@@ -91,13 +96,13 @@ def test_url(url, extraHeader):
 
 
 def convertBranch(repo_url, branch, extraHeader):
+    docfx_logger = get_logger(__name__)
     result = branch
     if not branch:
-        print(
-            "Using empty branch of {}, going to use master branch instead.".format(
+        msg = "Using empty branch of {}, going to use master branch instead.".format(
                 repo_url
             )
-        )
+        docfx_logger.info(msg)
         result = "master"
     if result == "master":
         check_params = ["git", "ls-remote", "--heads", repo_url, "refs/heads/main"]
@@ -109,11 +114,10 @@ def convertBranch(repo_url, branch, extraHeader):
 
         checkBr = subprocess.check_output(check_params)
         if checkBr:
-            print(
-                "Using master branch of {}, going to use main branch instead.".format(
+            msg = "Using master branch of {}, going to use main branch instead.".format(
                     repo_url
                 )
-            )
+            docfx_logger.info(msg)
             result = "main"
     return result
 
@@ -125,7 +129,7 @@ def checkout(folder, branch):
     Replacing
     https://apidrop.visualstudio.com/Content%20CI/_git/ReferenceAutomation?path=/Python/PyCommon.ps1&line=707&lineEnd=763&lineStartColumn=1&lineEndColumn=2&lineStyle=plain&_a=contents
     """
-
+    docfx_logger = get_logger(__name__)
     remote = "origin"
 
     if ":" in branch:
@@ -153,8 +157,8 @@ def checkout(folder, branch):
             )
 
         subprocess.run(["git", "-C", folder, "checkout", "--quiet", branch])
-
-        print("<CI INFO>: Switched to branch {}.".format(branch))
+        msg = "<CI INFO>: Switched to branch {}.".format(branch)
+        docfx_logger.info(msg)
 
 
 def commit_and_push(repo_location, folder, extra_token=None):

py2docfx/convert_prepare/package_info.py

@@ -1,6 +1,7 @@
 from enum import Enum
 import os
 import re
+from py2docfx.docfx_yaml.logger import get_logger
 from py2docfx.convert_prepare.source import Source
 from py2docfx.convert_prepare.package_info_extra_settings import extra_exclude_path_by_package
 
@@ -9,14 +10,13 @@ class PackageInfo:
         PYPI = 1
         SOURCE_CODE = 2
         DIST_FILE = 3
-
     path: Source
-
     def __init__(self) -> None:
         pass
-
+    
     @classmethod
     def report_error(cls, name, value, condition=None):
+        docfx_logger = get_logger(__name__)
         if condition:
             message = "When {0}, found unexpected property of {1}, loaded value is: {2}".format(
                 condition, name, value
@@ -25,7 +25,8 @@ class PackageInfo:
             message = "Found unexpected property of {0}, loaded value is: {1}".format(
                 name, value
             )
-        raise ValueError(message)
+        docfx_logger.error(message)
+        raise ValueError()
 
     @classmethod
     def parse_from(cls, dict, reading_required_packages=False):
@@ -59,12 +60,14 @@ class PackageInfo:
             package_info.folder = package_info_dict.get("folder", None)
 
             if not package_info.url:
+                docfx_logger = get_logger(__name__)
                 if not package_info.folder:
-                    raise ValueError(
-                        "When install_type is source_code, folder or url should be provided"
-                    )
+                    msg = "When install_type is source_code, url or folder should be provided"
+                    docfx_logger.error(msg)
+                    raise ValueError()
                 else:
-                    print(f'Read source code from local folder: {package_info.folder}')
+                    msg = f'Read source code from local folder: {package_info.folder}'
+                    docfx_logger.info(msg)
 
         prefer_source_distribution = package_info_dict.get(
             "prefer_source_distribution", False
@@ -140,10 +143,13 @@ class PackageInfo:
         return (packageInstallName, pipInstallExtraOptions)
 
     def get_exluded_command(self) -> []:
+        docfx_logger = get_logger(__name__)
         if hasattr(self, "path"):
             code_location = self.path.source_folder
         else:
-            raise ValueError("Should set source code location before build documents")
+            msg = "Should set source code location before build documents"
+            docfx_logger.error(msg)
+            raise ValueError()
         exclude_path = []
         if code_location:
             exclude_path.append(os.path.join(code_location, "build/*"))

py2docfx/convert_prepare/pip_utils.py

@@ -1,6 +1,8 @@
 import subprocess
 import asyncio
+import os
 from py2docfx import PACKAGE_ROOT
+from py2docfx.docfx_yaml.logger import get_logger, log_subprocess_ouput
 
 PYPI = "pypi"
 
@@ -10,10 +12,9 @@ pip_install_common_options = [
     "--no-compile",
     "--no-warn-conflicts",
     "--disable-pip-version-check",
-    "--verbose",
+    "--verbose"
 ]
 
-
 def download(package_name, path, extra_index_url=None, prefer_source_distribution=True):
     # Downloads a package from PyPI to the specified path using pip.
     download_param = ["pip", "download", "--dest", path, "--no-deps", package_name]
@@ -24,7 +25,9 @@ def download(package_name, path, extra_index_url=None, prefer_source_distributio
         download_param.append("--no-binary=:all:")
     else:
         download_param.append("--prefer-binary")
-    subprocess.run(download_param, check=True, cwd=PACKAGE_ROOT)
+    output = subprocess.run(download_param, check=True, cwd=PACKAGE_ROOT, capture_output=True, text=True)
+    docfx_logger = get_logger(__name__)
+    log_subprocess_ouput(output, docfx_logger)
 
 
 def install(package_name, options):
@@ -32,16 +35,34 @@ def install(package_name, options):
     install_param = "pip install {} {}".format(
         " ".join(pip_install_common_options + options), package_name
     ).split(" ")
-    subprocess.run(install_param, check=True, cwd=PACKAGE_ROOT)
-
+    output = subprocess.run(install_param, check=True, cwd=PACKAGE_ROOT, capture_output=True, text=True)
+    docfx_logger = get_logger(__name__)
+    log_subprocess_ouput(output, docfx_logger)
 
 def install_in_exe(exe_path, package_name, options):
     # Installs a package from PyPI using pip.
     install_param = [exe_path] + "-m pip install {} {}".format(
         " ".join(pip_install_common_options + options), package_name
     ).split(" ")
-    subprocess.run(install_param, check=True, cwd=PACKAGE_ROOT)
-
+    output = subprocess.run(install_param, check=True, cwd=PACKAGE_ROOT, capture_output=True, text=True)
+    docfx_logger = get_logger(__name__)
+    log_subprocess_ouput(output, docfx_logger)
+    
 async def install_in_exe_async(exe_path, package_name, options):
     pip_cmd = ["-m", "pip", "install"]+ pip_install_common_options + options + [package_name]
-    await(await asyncio.create_subprocess_exec(exe_path, *pip_cmd)).wait()
+    await run_async_subprocess(exe_path, pip_cmd)
+    # await(await asyncio.create_subprocess_exec(exe_path, *pip_cmd)).wait()
+    
+async def run_async_subprocess(exe_path, cmd):
+    process = await asyncio.create_subprocess_exec(
+        exe_path, *cmd,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE
+    )
+    stdout, stderr = await process.communicate()
+    docfx_logger = get_logger(__name__)
+    if process.returncode != 0:
+        docfx_logger.error(stderr.decode('utf-8'))
+        raise RuntimeError()
+    else:
+        docfx_logger.info(stdout.decode('utf-8'))

py2docfx/convert_prepare/post_process/merge_toc.py

@@ -2,10 +2,13 @@ from __future__ import annotations # Avoid A | B annotation break under <= py3.9
 import os
 import shutil
 
+from py2docfx.docfx_yaml.logger import get_logger
+
 TOC_FILE_PATH = "toc.yml"
 PACKAGE_TOC_FILE_PATH = "_build/docfx_yaml/toc.yml"
 def merge_toc(
         root_doc_path: str | os.PathLike, package_doc_path: str | os.PathLike):
+    docfx_logger = get_logger(__name__)
     root_toc_path = os.path.join(root_doc_path, TOC_FILE_PATH)
     package_toc_path = os.path.join(package_doc_path, PACKAGE_TOC_FILE_PATH)
 
@@ -18,7 +21,8 @@ def merge_toc(
             if not toc_content.endswith("\n"):
                 root_toc_handle.write("\n")
     else:
-        raise ValueError(f"TOC content empty: {package_toc_path}")
+        docfx_logger.error(f"TOC content empty: {package_toc_path}")
+        raise ValueError()
 
     # delete package toc.yml
     os.remove(package_toc_path)