py2docfx 0.1.11rc1996319__py3-none-any.whl → 0.1.12.dev2002521__py3-none-any.whl

py2docfx/__main__.py

@@ -36,9 +36,8 @@ async def donwload_package_generate_documents(
             py2docfxEnvironment.prepare_base_venv(required_package_list, github_token, ado_token))
 
     for idx, package in enumerate(package_info_list):
-        os.environ['PROCESSING_PACKAGE_NAME'] = package.name
         package_number = start_num + idx
-        py2docfx_logger = py2docfxLogger.get_package_logger(__name__)
+        py2docfx_logger = py2docfxLogger.get_logger(__name__)
         msg = f"Processing package {package.name}, env_prepare_tasks: {len(env_prepare_tasks)}"
         py2docfx_logger.info(msg)
 
@@ -47,11 +46,11 @@ async def donwload_package_generate_documents(
         except Exception as e:
             msg = f"Failed to setup venv for package {package.name}: {e}"
             py2docfx_logger.error(msg)
-            raise
+            raise e
 
-        generate_document(package, output_root,
+        await generate_document(package, output_root,
                           py2docfxEnvironment.get_base_venv_sphinx_build_path(),
-                          py2docfxEnvironment.get_venv_package_path(idx), 
+                          py2docfxEnvironment.get_venv_package_path(idx),
                           py2docfxEnvironment.get_base_venv_exe())
 
         merge_toc(YAML_OUTPUT_ROOT, package.path.yaml_output_folder)
@@ -80,7 +79,7 @@ async def donwload_package_generate_documents(
             msg = f"Removing venv {idx-py2docfxEnvironment.VENV_BUFFER}"
             py2docfx_logger.info(msg)
             await env_remove_tasks[idx-py2docfxEnvironment.VENV_BUFFER]
-
+    
     if output_doc_folder:
         move_root_toc_to_target(YAML_OUTPUT_ROOT, output_doc_folder)
     
@@ -88,17 +87,28 @@ async def donwload_package_generate_documents(
         if env_remove_tasks[idx] != None and not env_remove_tasks[idx].done():
             await env_remove_tasks[idx]
 
-def main(argv) -> int:
+def fishish_up():
+    warning_count, error_count = py2docfxLogger.get_warning_error_count()
+    py2docfxLogger.output_log_by_log_level()
+    print(f"Warning count: {warning_count}, Error count: {error_count}")
+    logging.shutdown()
+    
+async def main(argv) -> int:
     # TODO: may need to purge pip cache
     (package_info_list,
      required_package_list,
      github_token, ado_token,
      output_root, verbose,
      show_warning) = parse_command_line_args(argv)
-    
+
     clean_up_folder_list = [py2docfxEnvironment.VENV_DIR, DIST_TEMP, SOURCE_REPO, TARGET_REPO, LOG_FOLDER]
     temp_folder_clean_up(clean_up_folder_list)
     
+    # create log folder and package log folder
+    log_folder = os.path.join(PACKAGE_ROOT, LOG_FOLDER)
+    os.makedirs(log_folder)
+    os.makedirs(os.path.join(log_folder, "package_logs"))
+    
     py2docfxLogger.decide_global_log_level(verbose, show_warning)
     
     py2docfx_logger = py2docfxLogger.get_logger(__name__)
@@ -110,19 +120,18 @@ def main(argv) -> int:
     output_doc_folder = prepare_out_dir(output_root)
 
     try:
-        asyncio.run(donwload_package_generate_documents(
+        await donwload_package_generate_documents(
             package_info_list, output_root, output_doc_folder,
-            github_token, ado_token, required_package_list))
+            github_token, ado_token, required_package_list)
     except Exception as e:
         msg = f"An error occurred: {e}"
         py2docfx_logger.error(msg)
+        fishish_up()
+        asyncio.get_event_loop().stop()
         raise
     
-    warning_count, error_count = py2docfxLogger.get_warning_error_count()
-    py2docfxLogger.output_log_by_log_level()
-    print(f"Warning count: {warning_count}, Error count: {error_count}")
-    logging.shutdown()
+    fishish_up()
     return 0
 
 if __name__ == "__main__":
-    sys.exit(main(sys.argv[1:]))
+    sys.exit(asyncio.run(main(sys.argv[1:])))

py2docfx/convert_prepare/environment.py

@@ -22,7 +22,7 @@ PIP_INSTALL_VENV_COMMON_OPTIONS = [
     "--disable-pip-version-check",
 ]
 
-def install_converter_requirements(executable: str):
+async def install_converter_requirements(executable: str):
     """
     Install setuptools/sphinx/pyyaml/jinja2
     Replacing logic of
@@ -41,14 +41,14 @@ def install_converter_requirements(executable: str):
     for module in REQUIREMENT_MODULES:
         msg = f"<CI INFO>: Upgrading {module}..."
         py2docfx_logger.info(msg)
-        subprocess.run(
-            pip_install_cmd + [module] + pip_install_common_options, check=True
-        )
+        full_cmd = pip_install_cmd + [module] + pip_install_common_options
+        await run_async_subprocess(executable, full_cmd, py2docfx_logger)
 
 async def install_venv_requirements(venv_num: int):
     venv_exe = get_venv_exe(venv_num)
     pip_cmd = ["-m", "pip", "install", "--upgrade"]+ VENV_REQUIREMENT_MODULES
-    await run_async_subprocess(venv_exe, pip_cmd)
+    py2docfx_logger = get_logger(__name__)
+    await run_async_subprocess(venv_exe, pip_cmd, py2docfx_logger)
 
 def get_venv_path(venv_num: int) -> str:
     return os.path.join(PACKAGE_ROOT, VENV_DIR, "venv"+str(venv_num))
@@ -85,7 +85,8 @@ def get_base_venv_sphinx_build_path() -> str:
 
 async def install_converter_requirement_async(executable: str):
     pip_cmd = PIP_INSTALL_COMMAND + PIP_INSTALL_VENV_COMMON_OPTIONS + REQUIREMENT_MODULES
-    await run_async_subprocess(executable, pip_cmd)
+    py2docfx_logger = get_logger(__name__)
+    await run_async_subprocess(executable, pip_cmd, py2docfx_logger)
 
 async def install_required_packages(
         executable: str, required_package_list: list[PackageInfo], github_token: str, ado_token: str):
@@ -95,7 +96,8 @@ async def install_required_packages(
         #     get_source(package, idx, vststoken=ado_token, githubtoken=github_token)
         package_name, options = package.get_install_command()
         pip_cmd = PIP_INSTALL_COMMAND + PIP_INSTALL_VENV_COMMON_OPTIONS + options + [package_name]
-        await run_async_subprocess(executable, pip_cmd)
+        py2docfx_logger = get_logger(__name__)
+        await run_async_subprocess(executable, pip_cmd, py2docfx_logger)
 
 async def create_environment(venv_path: int):
     if os.name == 'nt':
@@ -125,7 +127,7 @@ async def prepare_venv(venv_num: int, package_info: PackageInfo, package_number:
     py2docfx_logger = get_logger(__name__)
     await create_environment(get_venv_path(venv_num))
     await install_venv_requirements(venv_num)
-    get_source(get_venv_exe(venv_num), package_info, package_number, vststoken=ado_token, githubtoken=github_token)
+    await get_source(get_venv_exe(venv_num), package_info, package_number, vststoken=ado_token, githubtoken=github_token)
     package_name, options = package_info.get_install_command()
     await pip_utils.install_in_exe_async(get_venv_exe(venv_num), package_name, options)
     msg = f"<CI INFO>: venv{venv_num} setup complete."
@@ -138,8 +140,9 @@ async def remove_environment(venv_num: int):
         msg = f"<CI INFO>: Removing venv{venv_num}..."
         py2docfx_logger.info(msg)
         # Create a subprocess to run the shell command for removing the directory
+        cmd = f'rm -rf {venv_path}' if os.name != 'nt' else f'rmdir /S /Q {venv_path}'
         process = await asyncio.create_subprocess_shell(
-            f'rm -rf {venv_path}' if os.name != 'nt' else f'rmdir /S /Q {venv_path}',
+            cmd,
             stdout=asyncio.subprocess.PIPE,
             stderr=asyncio.subprocess.PIPE
         )
@@ -150,4 +153,4 @@ async def remove_environment(venv_num: int):
         else:
             msg = f"<CI ERROR>: Failed to remove venv{venv_num}. Error: {stderr.decode()}"
             py2docfx_logger.error(msg)
-            raise RuntimeError()
+            raise subprocess.CalledProcessError(process.returncode, cmd, stdout, stderr)

py2docfx/convert_prepare/generate_document.py

@@ -10,8 +10,8 @@ from py2docfx.convert_prepare.subpackage import merge_subpackage_files
 
 CONF_TEMPLATE_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "conf_templates")
 
-def generate_document(pkg: PackageInfo, output_root: str | os.PathLike, sphinx_build_path: str, extra_package_path: str, executable=sys.executable):
-    py2docfx_logger = get_package_logger(__name__)
+async def generate_document(pkg: PackageInfo, output_root: str | os.PathLike, sphinx_build_path: str, extra_package_path: str, executable=sys.executable):
+    py2docfx_logger = get_package_logger(__name__, pkg.name)
     # Copy manual written RST from target doc repo
     package_paths = pkg.path
     if output_root:
@@ -28,12 +28,12 @@ def generate_document(pkg: PackageInfo, output_root: str | os.PathLike, sphinx_b
         exclude_paths = pkg.get_exluded_command()
 
         if pkg.install_type == pkg.InstallType.SOURCE_CODE and getattr(pkg, "branch", None):
-            checkout(package_paths.source_folder, pkg.branch)
+            await checkout(package_paths.source_folder, pkg.branch)
 
         msg = f"<CI INFO>: Generating RST files for {pkg.name}."
         py2docfx_logger.info(msg)
 
-        subpackages_rst_record = run_apidoc(package_paths.doc_folder, package_paths.source_folder,
+        subpackages_rst_record = await run_apidoc(pkg.name, package_paths.doc_folder, package_paths.source_folder,
                                             exclude_paths, pkg)
 
     msg = f"<CI INFO>: Listing RST files:"
@@ -45,7 +45,7 @@ def generate_document(pkg: PackageInfo, output_root: str | os.PathLike, sphinx_b
     py2docfx_logger.info(msg)
 
     generate_conf(pkg, package_paths.doc_folder, CONF_TEMPLATE_DIR)
-    run_converter(package_paths.doc_folder, package_paths.yaml_output_folder, sphinx_build_path, extra_package_path, executable=executable)
+    await run_converter(pkg.name, package_paths.doc_folder, package_paths.yaml_output_folder, sphinx_build_path, extra_package_path, executable=executable)
     
     subpackages_path_record = {}
     if pkg.build_in_subpackage:
@@ -53,6 +53,6 @@ def generate_document(pkg: PackageInfo, output_root: str | os.PathLike, sphinx_b
         for (subpackage_name, subpackage_path) in subpackages_rst_record.items():
             subpackage_yaml_path = os.path.join(subpackages_yaml_path, subpackage_name)
             subpackages_path_record[subpackage_name] = subpackage_yaml_path
-            run_converter(subpackage_path, subpackage_yaml_path, sphinx_build_path, extra_package_path, executable=executable)
+            await run_converter(pkg.name, subpackage_path, subpackage_yaml_path, sphinx_build_path, extra_package_path, executable=executable)
 
         merge_subpackage_files(subpackages_path_record, package_paths.yaml_output_folder, pkg.name)

py2docfx/convert_prepare/get_source.py

@@ -72,7 +72,7 @@ def update_package_info(executable: str, pkg: PackageInfo, source_folder: str):
         source_folder=source_folder, yaml_output_folder=yaml_output_folder, package_name=pkg.name
     )
 
-def get_source(executable: str, pkg: PackageInfo, cnt: int, vststoken=None, githubtoken=None):
+async def get_source(executable: str, pkg: PackageInfo, cnt: int, vststoken=None, githubtoken=None):
     py2docfx_logger = get_logger(__name__)
     path_cnt = str(cnt)
     dist_dir = path.join(DIST_TEMP, path_cnt)
@@ -81,7 +81,7 @@ def get_source(executable: str, pkg: PackageInfo, cnt: int, vststoken=None, gith
         if pkg.url:
             repo_folder = path.join(SOURCE_REPO, path_cnt)
             token = githubtoken if "github.com" in pkg.url else vststoken
-            source_folder = git.clone(
+            source_folder = await git.clone(
                 repo_location=pkg.url,
                 branch=pkg.branch,
                 folder=repo_folder,
@@ -94,7 +94,7 @@ def get_source(executable: str, pkg: PackageInfo, cnt: int, vststoken=None, gith
             sys.path.insert(0, source_folder)
     elif pkg.install_type == PackageInfo.InstallType.PYPI:
         full_name = pkg.get_combined_name_version()
-        pip_utils.download(
+        await pip_utils.download(
             full_name,
             dist_dir,
             extra_index_url=pkg.extra_index_url,
@@ -102,17 +102,17 @@ def get_source(executable: str, pkg: PackageInfo, cnt: int, vststoken=None, gith
         )
         # unpack the downloaded wheel file.
         downloaded_dist_file = path.join(dist_dir, os.listdir(dist_dir)[0])
-        pack.unpack_dist(downloaded_dist_file)
+        await pack.unpack_dist(pkg.name, downloaded_dist_file)
         os.remove(downloaded_dist_file)
         source_folder = path.join(
             path.dirname(downloaded_dist_file),
             os.listdir(dist_dir)[0]
         )
     elif pkg.install_type == PackageInfo.InstallType.DIST_FILE:
-        pip_utils.download(pkg.location, dist_dir, prefer_source_distribution=False)
+        await pip_utils.download(pkg.location, dist_dir, prefer_source_distribution=False)
         # unpack the downloaded dist file.
         downloaded_dist_file = path.join(dist_dir, os.listdir(dist_dir)[0])
-        pack.unpack_dist(downloaded_dist_file)
+        await pack.unpack_dist(pkg.name, downloaded_dist_file)
         os.remove(downloaded_dist_file)
         if downloaded_dist_file.endswith(".tar.gz"):
             downloaded_dist_file = downloaded_dist_file.rsplit(".", maxsplit=1)[
@@ -124,6 +124,6 @@ def get_source(executable: str, pkg: PackageInfo, cnt: int, vststoken=None, gith
     else:
         msg = f"Unknown install type: {pkg.install_type}"
         py2docfx_logger.error(msg)
-        raise ValueError()
+        raise ValueError(msg)
 
     update_package_info(executable, pkg, source_folder)

py2docfx/convert_prepare/git.py

@@ -1,11 +1,11 @@
 import re
 import subprocess
 
-from py2docfx.docfx_yaml.logger import get_logger, log_git_clone_subprocess_ouput
+from py2docfx.docfx_yaml.logger import get_logger, run_async_subprocess_without_executable
 
 repoMap = {}
 
-def clone(repo_location, branch, folder, extra_token=None):
+async def clone(repo_location, branch, folder, extra_token=None):
     """
     Clone a git repo to the folder
     When extra_token isn't None, add it to the command
@@ -22,7 +22,7 @@ def clone(repo_location, branch, folder, extra_token=None):
     if not test_url(repo_location, extra_token):
         msg = "Git repo address {} is not a valid URL.".format(repo_location)
         py2docfx_logger.error(msg)
-        raise ValueError()
+        raise ValueError(msg)
     else:
         # Remove http(s):// from url to record. Further avoid dup-clone.
         pureURL = re.sub("^\s*https?://", "", repo_location)
@@ -57,8 +57,7 @@ def clone(repo_location, branch, folder, extra_token=None):
                             extra_token
                         ),
                     ]
-                output = subprocess.run(clone_params, check=True, capture_output=True, text=True)
-                log_git_clone_subprocess_ouput(output, py2docfx_logger)
+                await run_async_subprocess_without_executable(clone_params, py2docfx_logger)
                 repoMap[pureURL] = folder
                 msg = "<CI INFO>: Repo {} successfully cloned in {}...".format(
                         repo_location, repoMap[pureURL]
@@ -123,7 +122,7 @@ def convertBranch(repo_url, branch, extraHeader):
     return result
 
 
-def checkout(folder, branch):
+async def checkout(folder, branch):
     """
     Checkout one folder to a given branch
 
@@ -145,8 +144,7 @@ def checkout(folder, branch):
     if "* {}".format(branch) not in branches:
         # branch is not exactly current branch
         if branch not in branches:
-            output = subprocess.run(
-                [
+            fetch_cmd = [
                     "git",
                     "-C",
                     folder,
@@ -154,12 +152,11 @@ def checkout(folder, branch):
                     "--quiet",
                     remote,
                     "{}:{}".format(branch, branch),
-                ], capture_output=True, text=True
-            )
-            log_subprocess_ouput(output, py2docfx_logger)
+                ]
+            await run_async_subprocess_without_executable(fetch_cmd, py2docfx_logger)
 
-        output = subprocess.run(["git", "-C", folder, "checkout", "--quiet", branch], check=True, capture_output=True, text=True)
-        log_subprocess_ouput(output, py2docfx_logger)
+        checkout_cmd = ["git", "-C", folder, "checkout", "--quiet", branch]
+        await run_async_subprocess_without_executable(checkout_cmd, py2docfx_logger)
         msg = "<CI INFO>: Switched to branch {}.".format(branch)
         py2docfx_logger.info(msg)
 

py2docfx/convert_prepare/install_package.py

@@ -1,6 +1,6 @@
 from py2docfx.convert_prepare.package_info import PackageInfo
 import py2docfx.convert_prepare.pip_utils as pip_utils
 
-def install_package(pkg: PackageInfo):
+async def install_package(pkg: PackageInfo):
     package_name, options = pkg.get_install_command()
-    pip_utils.install(package_name, options)
+    await pip_utils.install(package_name, options)

py2docfx/convert_prepare/pack.py

@@ -1,14 +1,13 @@
 import shutil
 from os import path
 import os
-import subprocess
 import sys
 
-from py2docfx.docfx_yaml.logger import get_package_logger, log_subprocess_ouput
+from py2docfx.docfx_yaml.logger import get_package_logger, run_async_subprocess
 
-def unpack_dist(dist_file):
+async def unpack_dist(package_name, dist_file):
     if dist_file.endswith(".whl"):
-        unpack_wheel(dist_file)
+        await unpack_wheel(package_name, dist_file)
     else:
         unpack_compressed(dist_file)
 
@@ -20,20 +19,18 @@ def unpack_compressed(file_path):
     shutil.unpack_archive(file_path, path.dirname(file_path))
 
 
-def unpack_wheel(file_path):
+async def unpack_wheel(package_name, file_path):
     """
     Transform a wheel file to a folder containing source code
     """
-    py2docfx_logger = get_package_logger(__name__)
-    command = [sys.executable,
-               '-m',
+    py2docfx_logger = get_package_logger(__name__, package_name)
+    command = ['-m',
                'wheel',
                'unpack',
                file_path,
                '-d',
                os.path.dirname(file_path)]
-    output = subprocess.run(command, check=True, capture_output=True, text=True)
-    log_subprocess_ouput(output, py2docfx_logger)
+    await run_async_subprocess(sys.executable, command, py2docfx_logger)
     extract_folder = [file for file in os.listdir(path.dirname(file_path)) if path.isdir(path.join(path.dirname(file_path), file))][0]
     data_folder = path.join(path.dirname(file_path), extract_folder, extract_folder + ".data")
     if path.exists(data_folder):

py2docfx/convert_prepare/package_info.py

@@ -26,7 +26,7 @@ class PackageInfo:
                 name, value
             )
         py2docfx_logger.error(message)
-        raise ValueError()
+        raise ValueError(message)
 
     @classmethod
     def parse_from(cls, dict, reading_required_packages=False):
@@ -64,7 +64,7 @@ class PackageInfo:
                 if not package_info.folder:
                     msg = "When install_type is source_code, url or folder should be provided"
                     py2docfx_logger.error(msg)
-                    raise ValueError()
+                    raise ValueError(msg)
                 else:
                     msg = f'Read source code from local folder: {package_info.folder}'
                     py2docfx_logger.info(msg)
@@ -149,7 +149,7 @@ class PackageInfo:
         else:
             msg = "Should set source code location before build documents"
             py2docfx_logger.error(msg)
-            raise ValueError()
+            raise ValueError(msg)
         exclude_path = []
         if code_location:
             exclude_path.append(os.path.join(code_location, "build/*"))

py2docfx/convert_prepare/pip_utils.py

@@ -1,7 +1,7 @@
-import subprocess
+import sys
 
 from py2docfx import PACKAGE_ROOT
-from py2docfx.docfx_yaml.logger import get_logger, log_subprocess_ouput, run_async_subprocess
+from py2docfx.docfx_yaml.logger import get_logger, run_async_subprocess, run_async_subprocess_without_executable
 
 PYPI = "pypi"
 
@@ -14,7 +14,7 @@ pip_install_common_options = [
     "--verbose"
 ]
 
-def download(package_name, path, extra_index_url=None, prefer_source_distribution=True):
+async def download(package_name, path, extra_index_url=None, prefer_source_distribution=True):
     # Downloads a package from PyPI to the specified path using pip.
     download_param = ["pip", "download", "--dest", path, "--no-deps", package_name]
     if extra_index_url:
@@ -24,29 +24,27 @@ def download(package_name, path, extra_index_url=None, prefer_source_distributio
         download_param.append("--no-binary=:all:")
     else:
         download_param.append("--prefer-binary")
-    output = subprocess.run(download_param, check=True, cwd=PACKAGE_ROOT, capture_output=True, text=True)
-    py2docfx_logger = get_logger(__name__)
-    log_subprocess_ouput(output, py2docfx_logger)
 
+    py2docfx_logger = get_logger(__name__)
+    await run_async_subprocess_without_executable(download_param, py2docfx_logger, cwd=PACKAGE_ROOT)
 
-def install(package_name, options):
+async def install(package_name, options):
     # Installs a package from PyPI using pip.
     install_param = "pip install {} {}".format(
         " ".join(pip_install_common_options + options), package_name
     ).split(" ")
-    output = subprocess.run(install_param, check=True, cwd=PACKAGE_ROOT, capture_output=True, text=True)
     py2docfx_logger = get_logger(__name__)
-    log_subprocess_ouput(output, py2docfx_logger)
+    await run_async_subprocess_without_executable(install_param, py2docfx_logger, cwd=PACKAGE_ROOT)
 
-def install_in_exe(exe_path, package_name, options):
+async def install_in_exe(exe_path, package_name, options):
     # Installs a package from PyPI using pip.
     install_param = [exe_path] + "-m pip install {} {}".format(
         " ".join(pip_install_common_options + options), package_name
     ).split(" ")
-    output = subprocess.run(install_param, check=True, cwd=PACKAGE_ROOT, capture_output=True, text=True)
     py2docfx_logger = get_logger(__name__)
-    log_subprocess_ouput(output, py2docfx_logger)
-    
+    await run_async_subprocess(exe_path, install_param, py2docfx_logger, cwd=PACKAGE_ROOT)
+
 async def install_in_exe_async(exe_path, package_name, options):
     pip_cmd = ["-m", "pip", "install"]+ pip_install_common_options + options + [package_name]
-    await run_async_subprocess(exe_path, pip_cmd)
+    py2docfx_logger = get_logger(__name__)
+    await run_async_subprocess(exe_path, pip_cmd, py2docfx_logger)

py2docfx/convert_prepare/post_process/merge_toc.py

@@ -21,8 +21,9 @@ def merge_toc(
             if not toc_content.endswith("\n"):
                 root_toc_handle.write("\n")
     else:
-        py2docfx_logger.error(f"TOC content empty: {package_toc_path}")
-        raise ValueError()
+        msg = f"TOC content empty: {package_toc_path}"
+        py2docfx_logger.error(msg)
+        raise ValueError(msg)
 
     # delete package toc.yml
     os.remove(package_toc_path)

py2docfx/convert_prepare/sphinx_caller.py

@@ -1,9 +1,10 @@
+import asyncio
 import os
-import sys
 import subprocess
+import sys
 
 from py2docfx import PACKAGE_ROOT
-from py2docfx.docfx_yaml.logger import get_package_logger,log_subprocess_ouput
+from py2docfx.docfx_yaml.logger import get_package_logger,run_async_subprocess
 from py2docfx.convert_prepare.package_info import PackageInfo
 from py2docfx.convert_prepare.paths import folder_is_hidden
 from py2docfx.convert_prepare.subpackage import (get_subpackages,
@@ -11,14 +12,14 @@ from py2docfx.convert_prepare.subpackage import (get_subpackages,
 
 DEBUG_SPHINX_FLAG = 'PY2DOCFX_DEBUG_SPHINX'
 
-def run_apidoc(rst_path, source_code_path, exclude_paths, package_info: PackageInfo):
+async def run_apidoc(package_name, rst_path, source_code_path, exclude_paths, package_info: PackageInfo):
     """
     Run sphinx-apidoc to generate RST inside rst_path folder
 
     Replacing
     https://apidrop.visualstudio.com/Content%20CI/_git/ReferenceAutomation?path=/Python/build.ps1&line=110&lineEnd=126&lineStartColumn=1&lineEndColumn=14&lineStyle=plain&_a=contents
     """
-    py2docfx_logger = get_package_logger(__name__)
+    py2docfx_logger = get_package_logger(__name__, package_name)
     subfolderList = [name for name in
                        os.listdir(source_code_path)
                        if os.path.isdir(os.path.join(source_code_path, name))
@@ -40,8 +41,8 @@ def run_apidoc(rst_path, source_code_path, exclude_paths, package_info: PackageI
                 subfolderPath,
             ]
             args.extend(exclude_paths)
-            output = subprocess.run([sys.executable, "-m", "sphinx.ext.apidoc"] + args, capture_output=True, text=True)
-            log_subprocess_ouput(output, py2docfx_logger)
+            full_args = ["-m", "sphinx.ext.apidoc"] + args
+            await run_async_subprocess(sys.executable, full_args, py2docfx_logger)
             if package_info.build_in_subpackage and subfolder == "azure":
                 subpackages_rst_record = move_rst_files_to_subfolder(
                     package_paths.doc_folder, package_info.name,
@@ -49,7 +50,13 @@ def run_apidoc(rst_path, source_code_path, exclude_paths, package_info: PackageI
     return subpackages_rst_record
 
 
-def run_converter(rst_path, out_path, sphinx_build_path: str, extra_package_path: str, conf_path = None, executable = sys.executable):
+async def run_converter(package_name: str,
+                    rst_path,
+                    out_path,
+                    sphinx_build_path: str,
+                    extra_package_path: str,
+                    conf_path = None,
+                    executable = sys.executable):
     """
     Take rst files as input and run sphinx converter
 
@@ -58,7 +65,7 @@ def run_converter(rst_path, out_path, sphinx_build_path: str, extra_package_path
     Replacing
     https://apidrop.visualstudio.com/Content%20CI/_git/ReferenceAutomation?path=/Python/build.ps1&line=150&lineEnd=161&lineStartColumn=13&lineEndColumn=52&lineStyle=plain&_a=contents
     """
-    py2docfx_logger = get_package_logger(__name__)
+    py2docfx_logger = get_package_logger(__name__, package_name)
     outdir = os.path.join(out_path, "_build")
 
     # Sphinx/docutils have memory leak including linecaches, module-import-caches,
@@ -67,9 +74,8 @@ def run_converter(rst_path, out_path, sphinx_build_path: str, extra_package_path
     if not sys.executable:
         msg = "Can't get the executable binary for the Python interpreter."
         py2docfx_logger.error(msg)
-        raise ValueError()
+        raise ValueError(msg)
     sphinx_param = [
-        executable,
         sphinx_build_path,
         rst_path,
         outdir,
@@ -86,6 +92,22 @@ def run_converter(rst_path, out_path, sphinx_build_path: str, extra_package_path
         env_tmp["PYTHONPATH"] = f"{extra_package_path};{package_root_parent};"
     else:
         env_tmp["PYTHONPATH"] = f"{extra_package_path}:{package_root_parent}:"
-    output = subprocess.run(sphinx_param, check=True, cwd=PACKAGE_ROOT, env=env_tmp, capture_output=True ,text=True)
-    log_subprocess_ouput(output, py2docfx_logger)
+    os.environ['PROCESSING_PACKAGE_NAME'] = package_name
+    proc = await asyncio.create_subprocess_exec(
+        executable, *sphinx_param,
+        cwd=PACKAGE_ROOT,
+        env=env_tmp,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE
+    )
+    stdout, stderr = await proc.communicate()
+    return_code = proc.returncode
+
+    if return_code == 0:
+        py2docfx_logger.info(f"{stdout}")
+        py2docfx_logger.info(f"{stderr}")
+    else:
+        py2docfx_logger.error(f"{stderr}")
+        raise subprocess.CalledProcessError(return_code, sphinx_param, stdout, stderr)
+
     return outdir

py2docfx/convert_prepare/tests/test_environment.py

@@ -1,8 +1,6 @@
-import asyncio
 import os
 import shutil
 import pytest
-import sys
 
 from py2docfx.convert_prepare.environment import create_environment
 from py2docfx.convert_prepare.environment import remove_environment
@@ -14,7 +12,6 @@ from py2docfx.convert_prepare.environment import prepare_base_venv
 from py2docfx.convert_prepare.environment import get_base_venv_exe
 from py2docfx.convert_prepare.environment import get_base_venv_path
 from py2docfx.convert_prepare.package_info import PackageInfo
-pytest_plugins = ('pytest_asyncio',)
 
 @pytest.mark.asyncio
 async def test_venv_creation_package_install_remove():

py2docfx/convert_prepare/tests/test_generate_document.py

@@ -4,6 +4,7 @@ Test the generate_document function.
 import os
 import sys
 import shutil
+import pytest
 import sphinx
 import sphinx.cmd.build
 
@@ -11,7 +12,8 @@ from py2docfx.convert_prepare.generate_document import generate_document
 from py2docfx.convert_prepare.package_info import PackageInfo
 from py2docfx.convert_prepare.source import Source
 
-def test_generate_document(tmp_path):
+@pytest.mark.asyncio
+async def test_generate_document(tmp_path):
     """
     Test the generate_document function.
     """
@@ -42,7 +44,7 @@ def test_generate_document(tmp_path):
 
     # call the function
     
-    generate_document(package, output_root, sphinx_build_path = sphinx.cmd.build.__file__, extra_package_path = source_folder)
+    await generate_document(package, output_root, sphinx_build_path = sphinx.cmd.build.__file__, extra_package_path = source_folder)
 
     #assert the result
     yaml_path = os.path.join(yaml_output_folder, "_build", "docfx_yaml")