py2docfx 0.1.11.dev1994698__py3-none-any.whl → 0.1.11.dev1994975__py3-none-any.whl

py2docfx/convert_prepare/environment.py

@@ -22,7 +22,7 @@ PIP_INSTALL_VENV_COMMON_OPTIONS = [
     "--disable-pip-version-check",
 ]
 
-def install_converter_requirements(executable: str):
+async def install_converter_requirements(executable: str):
     """
     Install setuptools/sphinx/pyyaml/jinja2
     Replacing logic of
@@ -41,14 +41,14 @@ def install_converter_requirements(executable: str):
     for module in REQUIREMENT_MODULES:
         msg = f"<CI INFO>: Upgrading {module}..."
         py2docfx_logger.info(msg)
-        subprocess.run(
-            pip_install_cmd + [module] + pip_install_common_options, check=True
-        )
+        full_cmd = pip_install_cmd + [module] + pip_install_common_options
+        await run_async_subprocess(executable, full_cmd, py2docfx_logger)
 
 async def install_venv_requirements(venv_num: int):
     venv_exe = get_venv_exe(venv_num)
     pip_cmd = ["-m", "pip", "install", "--upgrade"]+ VENV_REQUIREMENT_MODULES
-    await run_async_subprocess(venv_exe, pip_cmd)
+    py2docfx_logger = get_logger(__name__)
+    await run_async_subprocess(venv_exe, pip_cmd, py2docfx_logger)
 
 def get_venv_path(venv_num: int) -> str:
     return os.path.join(PACKAGE_ROOT, VENV_DIR, "venv"+str(venv_num))
@@ -85,7 +85,8 @@ def get_base_venv_sphinx_build_path() -> str:
 
 async def install_converter_requirement_async(executable: str):
     pip_cmd = PIP_INSTALL_COMMAND + PIP_INSTALL_VENV_COMMON_OPTIONS + REQUIREMENT_MODULES
-    await run_async_subprocess(executable, pip_cmd)
+    py2docfx_logger = get_logger(__name__)
+    await run_async_subprocess(executable, pip_cmd, py2docfx_logger)
 
 async def install_required_packages(
         executable: str, required_package_list: list[PackageInfo], github_token: str, ado_token: str):
@@ -95,7 +96,8 @@ async def install_required_packages(
         #     get_source(package, idx, vststoken=ado_token, githubtoken=github_token)
         package_name, options = package.get_install_command()
         pip_cmd = PIP_INSTALL_COMMAND + PIP_INSTALL_VENV_COMMON_OPTIONS + options + [package_name]
-        await run_async_subprocess(executable, pip_cmd)
+        py2docfx_logger = get_logger(__name__)
+        await run_async_subprocess(executable, pip_cmd, py2docfx_logger)
 
 async def create_environment(venv_path: int):
     if os.name == 'nt':
@@ -125,7 +127,7 @@ async def prepare_venv(venv_num: int, package_info: PackageInfo, package_number:
     py2docfx_logger = get_logger(__name__)
     await create_environment(get_venv_path(venv_num))
     await install_venv_requirements(venv_num)
-    get_source(get_venv_exe(venv_num), package_info, package_number, vststoken=ado_token, githubtoken=github_token)
+    await get_source(get_venv_exe(venv_num), package_info, package_number, vststoken=ado_token, githubtoken=github_token)
     package_name, options = package_info.get_install_command()
     await pip_utils.install_in_exe_async(get_venv_exe(venv_num), package_name, options)
     msg = f"<CI INFO>: venv{venv_num} setup complete."
@@ -138,8 +140,9 @@ async def remove_environment(venv_num: int):
         msg = f"<CI INFO>: Removing venv{venv_num}..."
         py2docfx_logger.info(msg)
         # Create a subprocess to run the shell command for removing the directory
+        cmd = f'rm -rf {venv_path}' if os.name != 'nt' else f'rmdir /S /Q {venv_path}'
         process = await asyncio.create_subprocess_shell(
-            f'rm -rf {venv_path}' if os.name != 'nt' else f'rmdir /S /Q {venv_path}',
+            cmd,
             stdout=asyncio.subprocess.PIPE,
             stderr=asyncio.subprocess.PIPE
         )
@@ -150,4 +153,4 @@ async def remove_environment(venv_num: int):
         else:
             msg = f"<CI ERROR>: Failed to remove venv{venv_num}. Error: {stderr.decode()}"
             py2docfx_logger.error(msg)
-            raise RuntimeError(msg)
+            raise subprocess.CalledProcessError(process.returncode, cmd, stdout, stderr)

py2docfx/convert_prepare/generate_document.py

@@ -28,12 +28,12 @@ async def generate_document(pkg: PackageInfo, output_root: str | os.PathLike, sp
         exclude_paths = pkg.get_exluded_command()
 
         if pkg.install_type == pkg.InstallType.SOURCE_CODE and getattr(pkg, "branch", None):
-            checkout(package_paths.source_folder, pkg.branch)
+            await checkout(package_paths.source_folder, pkg.branch)
 
         msg = f"<CI INFO>: Generating RST files for {pkg.name}."
         py2docfx_logger.info(msg)
 
-        subpackages_rst_record = run_apidoc(pkg.name, package_paths.doc_folder, package_paths.source_folder,
+        subpackages_rst_record = await run_apidoc(pkg.name, package_paths.doc_folder, package_paths.source_folder,
                                             exclude_paths, pkg)
 
     msg = f"<CI INFO>: Listing RST files:"

py2docfx/convert_prepare/get_source.py

@@ -72,7 +72,7 @@ def update_package_info(executable: str, pkg: PackageInfo, source_folder: str):
         source_folder=source_folder, yaml_output_folder=yaml_output_folder, package_name=pkg.name
     )
 
-def get_source(executable: str, pkg: PackageInfo, cnt: int, vststoken=None, githubtoken=None):
+async def get_source(executable: str, pkg: PackageInfo, cnt: int, vststoken=None, githubtoken=None):
     py2docfx_logger = get_logger(__name__)
     path_cnt = str(cnt)
     dist_dir = path.join(DIST_TEMP, path_cnt)
@@ -81,7 +81,7 @@ def get_source(executable: str, pkg: PackageInfo, cnt: int, vststoken=None, gith
         if pkg.url:
             repo_folder = path.join(SOURCE_REPO, path_cnt)
             token = githubtoken if "github.com" in pkg.url else vststoken
-            source_folder = git.clone(
+            source_folder = await git.clone(
                 repo_location=pkg.url,
                 branch=pkg.branch,
                 folder=repo_folder,
@@ -94,7 +94,7 @@ def get_source(executable: str, pkg: PackageInfo, cnt: int, vststoken=None, gith
             sys.path.insert(0, source_folder)
     elif pkg.install_type == PackageInfo.InstallType.PYPI:
         full_name = pkg.get_combined_name_version()
-        pip_utils.download(
+        await pip_utils.download(
             full_name,
             dist_dir,
             extra_index_url=pkg.extra_index_url,
@@ -102,17 +102,17 @@ def get_source(executable: str, pkg: PackageInfo, cnt: int, vststoken=None, gith
         )
         # unpack the downloaded wheel file.
         downloaded_dist_file = path.join(dist_dir, os.listdir(dist_dir)[0])
-        pack.unpack_dist(pkg.name, downloaded_dist_file)
+        await pack.unpack_dist(pkg.name, downloaded_dist_file)
         os.remove(downloaded_dist_file)
         source_folder = path.join(
             path.dirname(downloaded_dist_file),
             os.listdir(dist_dir)[0]
         )
     elif pkg.install_type == PackageInfo.InstallType.DIST_FILE:
-        pip_utils.download(pkg.location, dist_dir, prefer_source_distribution=False)
+        await pip_utils.download(pkg.location, dist_dir, prefer_source_distribution=False)
         # unpack the downloaded dist file.
         downloaded_dist_file = path.join(dist_dir, os.listdir(dist_dir)[0])
-        pack.unpack_dist(pkg.name, downloaded_dist_file)
+        await pack.unpack_dist(pkg.name, downloaded_dist_file)
         os.remove(downloaded_dist_file)
         if downloaded_dist_file.endswith(".tar.gz"):
             downloaded_dist_file = downloaded_dist_file.rsplit(".", maxsplit=1)[

py2docfx/convert_prepare/git.py

@@ -1,11 +1,11 @@
 import re
 import subprocess
 
-from py2docfx.docfx_yaml.logger import get_logger, log_git_clone_subprocess_ouput
+from py2docfx.docfx_yaml.logger import get_logger, run_async_subprocess_without_executable
 
 repoMap = {}
 
-def clone(repo_location, branch, folder, extra_token=None):
+async def clone(repo_location, branch, folder, extra_token=None):
     """
     Clone a git repo to the folder
     When extra_token isn't None, add it to the command
@@ -57,8 +57,7 @@ def clone(repo_location, branch, folder, extra_token=None):
                             extra_token
                         ),
                     ]
-                output = subprocess.run(clone_params, check=True, capture_output=True, text=True)
-                log_git_clone_subprocess_ouput(output, py2docfx_logger)
+                await run_async_subprocess_without_executable(clone_params, py2docfx_logger)
                 repoMap[pureURL] = folder
                 msg = "<CI INFO>: Repo {} successfully cloned in {}...".format(
                         repo_location, repoMap[pureURL]
@@ -123,7 +122,7 @@ def convertBranch(repo_url, branch, extraHeader):
     return result
 
 
-def checkout(folder, branch):
+async def checkout(folder, branch):
     """
     Checkout one folder to a given branch
 
@@ -145,8 +144,7 @@ def checkout(folder, branch):
     if "* {}".format(branch) not in branches:
         # branch is not exactly current branch
         if branch not in branches:
-            output = subprocess.run(
-                [
+            fetch_cmd = [
                     "git",
                     "-C",
                     folder,
@@ -154,12 +152,11 @@ def checkout(folder, branch):
                     "--quiet",
                     remote,
                     "{}:{}".format(branch, branch),
-                ], capture_output=True, text=True
-            )
-            log_subprocess_ouput(output, py2docfx_logger)
+                ]
+            await run_async_subprocess_without_executable(fetch_cmd, py2docfx_logger)
 
-        output = subprocess.run(["git", "-C", folder, "checkout", "--quiet", branch], check=True, capture_output=True, text=True)
-        log_subprocess_ouput(output, py2docfx_logger)
+        checkout_cmd = ["git", "-C", folder, "checkout", "--quiet", branch]
+        await run_async_subprocess_without_executable(checkout_cmd, py2docfx_logger)
         msg = "<CI INFO>: Switched to branch {}.".format(branch)
         py2docfx_logger.info(msg)
 

py2docfx/convert_prepare/install_package.py

@@ -1,6 +1,6 @@
 from py2docfx.convert_prepare.package_info import PackageInfo
 import py2docfx.convert_prepare.pip_utils as pip_utils
 
-def install_package(pkg: PackageInfo):
+async def install_package(pkg: PackageInfo):
     package_name, options = pkg.get_install_command()
-    pip_utils.install(package_name, options)
+    await pip_utils.install(package_name, options)

py2docfx/convert_prepare/pack.py

@@ -1,14 +1,13 @@
 import shutil
 from os import path
 import os
-import subprocess
 import sys
 
-from py2docfx.docfx_yaml.logger import get_package_logger, log_subprocess_ouput
+from py2docfx.docfx_yaml.logger import get_package_logger, run_async_subprocess
 
-def unpack_dist(package_name, dist_file):
+async def unpack_dist(package_name, dist_file):
     if dist_file.endswith(".whl"):
-        unpack_wheel(package_name, dist_file)
+        await unpack_wheel(package_name, dist_file)
     else:
         unpack_compressed(dist_file)
 
@@ -20,20 +19,18 @@ def unpack_compressed(file_path):
     shutil.unpack_archive(file_path, path.dirname(file_path))
 
 
-def unpack_wheel(package_name, file_path):
+async def unpack_wheel(package_name, file_path):
     """
     Transform a wheel file to a folder containing source code
     """
     py2docfx_logger = get_package_logger(__name__, package_name)
-    command = [sys.executable,
-               '-m',
+    command = ['-m',
                'wheel',
                'unpack',
                file_path,
                '-d',
                os.path.dirname(file_path)]
-    output = subprocess.run(command, check=True, capture_output=True, text=True)
-    log_subprocess_ouput(output, py2docfx_logger)
+    await run_async_subprocess(sys.executable, command, py2docfx_logger)
     extract_folder = [file for file in os.listdir(path.dirname(file_path)) if path.isdir(path.join(path.dirname(file_path), file))][0]
     data_folder = path.join(path.dirname(file_path), extract_folder, extract_folder + ".data")
     if path.exists(data_folder):

py2docfx/convert_prepare/pip_utils.py

@@ -1,7 +1,7 @@
-import subprocess
+import sys
 
 from py2docfx import PACKAGE_ROOT
-from py2docfx.docfx_yaml.logger import get_logger, log_subprocess_ouput, run_async_subprocess
+from py2docfx.docfx_yaml.logger import get_logger, run_async_subprocess, run_async_subprocess_without_executable
 
 PYPI = "pypi"
 
@@ -14,7 +14,7 @@ pip_install_common_options = [
     "--verbose"
 ]
 
-def download(package_name, path, extra_index_url=None, prefer_source_distribution=True):
+async def download(package_name, path, extra_index_url=None, prefer_source_distribution=True):
     # Downloads a package from PyPI to the specified path using pip.
     download_param = ["pip", "download", "--dest", path, "--no-deps", package_name]
     if extra_index_url:
@@ -24,29 +24,27 @@ def download(package_name, path, extra_index_url=None, prefer_source_distributio
         download_param.append("--no-binary=:all:")
     else:
         download_param.append("--prefer-binary")
-    output = subprocess.run(download_param, check=True, cwd=PACKAGE_ROOT, capture_output=True, text=True)
-    py2docfx_logger = get_logger(__name__)
-    log_subprocess_ouput(output, py2docfx_logger)
 
+    py2docfx_logger = get_logger(__name__)
+    await run_async_subprocess_without_executable(download_param, py2docfx_logger, cwd=PACKAGE_ROOT)
 
-def install(package_name, options):
+async def install(package_name, options):
     # Installs a package from PyPI using pip.
     install_param = "pip install {} {}".format(
         " ".join(pip_install_common_options + options), package_name
     ).split(" ")
-    output = subprocess.run(install_param, check=True, cwd=PACKAGE_ROOT, capture_output=True, text=True)
     py2docfx_logger = get_logger(__name__)
-    log_subprocess_ouput(output, py2docfx_logger)
+    await run_async_subprocess_without_executable(install_param, py2docfx_logger, cwd=PACKAGE_ROOT)
 
-def install_in_exe(exe_path, package_name, options):
+async def install_in_exe(exe_path, package_name, options):
     # Installs a package from PyPI using pip.
     install_param = [exe_path] + "-m pip install {} {}".format(
         " ".join(pip_install_common_options + options), package_name
     ).split(" ")
-    output = subprocess.run(install_param, check=True, cwd=PACKAGE_ROOT, capture_output=True, text=True)
     py2docfx_logger = get_logger(__name__)
-    log_subprocess_ouput(output, py2docfx_logger)
-    
+    await run_async_subprocess(exe_path, install_param, py2docfx_logger, cwd=PACKAGE_ROOT)
+
 async def install_in_exe_async(exe_path, package_name, options):
     pip_cmd = ["-m", "pip", "install"]+ pip_install_common_options + options + [package_name]
-    await run_async_subprocess(exe_path, pip_cmd)
+    py2docfx_logger = get_logger(__name__)
+    await run_async_subprocess(exe_path, pip_cmd, py2docfx_logger)

py2docfx/convert_prepare/sphinx_caller.py

@@ -4,7 +4,7 @@ import subprocess
 import sys
 
 from py2docfx import PACKAGE_ROOT
-from py2docfx.docfx_yaml.logger import get_package_logger,log_subprocess_ouput
+from py2docfx.docfx_yaml.logger import get_package_logger,run_async_subprocess
 from py2docfx.convert_prepare.package_info import PackageInfo
 from py2docfx.convert_prepare.paths import folder_is_hidden
 from py2docfx.convert_prepare.subpackage import (get_subpackages,
@@ -12,7 +12,7 @@ from py2docfx.convert_prepare.subpackage import (get_subpackages,
 
 DEBUG_SPHINX_FLAG = 'PY2DOCFX_DEBUG_SPHINX'
 
-def run_apidoc(package_name, rst_path, source_code_path, exclude_paths, package_info: PackageInfo):
+async def run_apidoc(package_name, rst_path, source_code_path, exclude_paths, package_info: PackageInfo):
     """
     Run sphinx-apidoc to generate RST inside rst_path folder
 
@@ -41,8 +41,8 @@ def run_apidoc(package_name, rst_path, source_code_path, exclude_paths, package_
                 subfolderPath,
             ]
             args.extend(exclude_paths)
-            output = subprocess.run([sys.executable, "-m", "sphinx.ext.apidoc"] + args, capture_output=True, text=True)
-            log_subprocess_ouput(output, py2docfx_logger)
+            full_args = ["-m", "sphinx.ext.apidoc"] + args
+            await run_async_subprocess(sys.executable, full_args, py2docfx_logger)
             if package_info.build_in_subpackage and subfolder == "azure":
                 subpackages_rst_record = move_rst_files_to_subfolder(
                     package_paths.doc_folder, package_info.name,

py2docfx/convert_prepare/tests/test_get_source.py

@@ -62,7 +62,8 @@ def test_update_package_info(init_package_info):
     assert package.name == "mock_package"
     assert package.version == "1.2.0"
 
-def test_get_source_git_clone(init_package_info):
+@pytest.mark.asyncio
+async def test_get_source_git_clone(init_package_info):
     """
     Test the git clone of get_source
     """
@@ -77,10 +78,11 @@ def test_get_source_git_clone(init_package_info):
     package.branch = "main"
     package.folder = None
     package.url = "https://github.com/Azure/azure-iot-hub-python"
-    get_source.get_source(sys.executable, package, 0)
+    await get_source.get_source(sys.executable, package, 0)
     assert git.status("source_repo/0") is True
 
-def test_get_source_dist_file_zip(init_package_info):
+@pytest.mark.asyncio
+async def test_get_source_dist_file_zip(init_package_info):
     """
     Test the zip dist file download of get_source
     """
@@ -93,12 +95,13 @@ def test_get_source_dist_file_zip(init_package_info):
     package.extra_index_url = None
     package.prefer_source_distribution = True
     package.location = "https://files.pythonhosted.org/packages/3e/71/f6f71a276e2e69264a97ad39ef850dca0a04fce67b12570730cb38d0ccac/azure-common-1.1.28.zip"
-    get_source.get_source(sys.executable, package, 1)
+    await get_source.get_source(sys.executable, package, 1)
     assert os.path.exists("dist_temp/1/azure-common-1.1.28")
     assert package.path.source_folder == os.path.join("dist_temp", "1", "azure-common-1.1.28")
     assert os.path.exists("dist_temp/1/azure-common-1.1.28.zip") is False
 
-def test_get_source_dist_file_whl(init_package_info):
+@pytest.mark.asyncio
+async def test_get_source_dist_file_whl(init_package_info):
     """
     Test the whl dist file download of get_source
     """
@@ -111,12 +114,13 @@ def test_get_source_dist_file_whl(init_package_info):
     package.extra_index_url = None
     package.prefer_source_distribution = True
     package.location = "https://files.pythonhosted.org/packages/62/55/7f118b9c1b23ec15ca05d15a578d8207aa1706bc6f7c87218efffbbf875d/azure_common-1.1.28-py2.py3-none-any.whl"
-    get_source.get_source(sys.executable, package, 2)
+    await get_source.get_source(sys.executable, package, 2)
     assert os.path.exists("dist_temp/2/azure_common-1.1.28")
     assert package.path.source_folder == os.path.join("dist_temp", "2", "azure_common-1.1.28")
     assert os.path.exists("dist_temp/2/azure_common-1.1.28-py2.py3-none-any.whl") is False
 
-def test_get_source_dist_file_tar(init_package_info):
+@pytest.mark.asyncio
+async def test_get_source_dist_file_tar(init_package_info):
     """
     Test the tar dist file download of get_source
     """
@@ -129,12 +133,13 @@ def test_get_source_dist_file_tar(init_package_info):
     package.extra_index_url = None
     package.prefer_source_distribution = True
     package.location = "https://files.pythonhosted.org/packages/fa/19/43a9eb812b4d6071fdc2c55640318f7eb5a1be8dbd3b6f9d96a1996e1bb6/azure-core-1.29.4.tar.gz"
-    get_source.get_source(sys.executable, package, 3)
+    await get_source.get_source(sys.executable, package, 3)
     assert os.path.exists("dist_temp/3/azure-core-1.29.4")
     assert package.path.source_folder == os.path.join("dist_temp", "3", "azure-core-1.29.4")
     assert os.path.exists("dist_temp/3/azure-core-1.29.4.tar.gz") is False
 
-def test_get_source_pip_whl(init_package_info):
+@pytest.mark.asyncio
+async def test_get_source_pip_whl(init_package_info):
     """
     Test the pip install of get_source with prefer_source_distribution = False
     """
@@ -146,12 +151,13 @@ def test_get_source_pip_whl(init_package_info):
     package.build_in_subpackage = False
     package.extra_index_url = None
     package.prefer_source_distribution = False
-    get_source.get_source(sys.executable, package, 4)
+    await get_source.get_source(sys.executable, package, 4)
     assert os.path.exists("dist_temp/4/azure_common-1.1.28")
     assert package.path.source_folder == os.path.join("dist_temp", "4", "azure_common-1.1.28")
     assert os.path.exists("dist_temp/4/azure_common-1.1.28-py2.py3-none-any.whl") is False
 
-def test_get_source_pip_zip(init_package_info):
+@pytest.mark.asyncio
+async def test_get_source_pip_zip(init_package_info):
     """
     Test the pip install of get_source with prefer_source_distribution = True
     """
@@ -163,12 +169,13 @@ def test_get_source_pip_zip(init_package_info):
     package.build_in_subpackage = False
     package.extra_index_url = None
     package.prefer_source_distribution = True
-    get_source.get_source(sys.executable, package, 5)
+    await get_source.get_source(sys.executable, package, 5)
     assert os.path.exists("dist_temp/5/azure-common-1.1.28")
     assert package.path.source_folder == os.path.join("dist_temp", "5", "azure-common-1.1.28")
     assert os.path.exists("dist_temp/5/azure-common-1.1.28.zip") is False
 
-def test_get_source_zip_file_at_position_0(init_package_info):
+@pytest.mark.asyncio
+async def test_get_source_zip_file_at_position_0(init_package_info):
     """
     Test the pip install of packages with zip or tar file at position 0 in the dirctory list
     """
@@ -180,7 +187,7 @@ def test_get_source_zip_file_at_position_0(init_package_info):
     package.build_in_subpackage = False
     package.extra_index_url = None
     package.prefer_source_distribution = True
-    get_source.get_source(sys.executable, package, 6)
+    await get_source.get_source(sys.executable, package, 6)
     assert os.path.exists("dist_temp/6/azure_template-0.1.0b3942895")
     assert package.path.source_folder == os.path.join("dist_temp", "6", "azure_template-0.1.0b3942895")
     assert os.path.exists("dist_temp/6/azure_template-0.1.0b3942895.tar.gz") is False

py2docfx/convert_prepare/tests/test_pack.py

@@ -5,6 +5,7 @@ import subprocess
 import shutil
 import glob
 from os import path
+import pytest
 
 from py2docfx.convert_prepare.pack import unpack_compressed, unpack_wheel
 
@@ -82,8 +83,8 @@ def test_pack_unpack_compressed(tmp_path):
         path.abspath("convert_prepare/tests/data/pack"), tmp_path / "gz", [], [str(gz_file_path)]
     )
 
-
-def test_pack_unpack_wheel(tmp_path):
+@pytest.mark.asyncio
+async def test_pack_unpack_wheel(tmp_path):
     def _prepare_wheel(target_path):
         subprocess.run(
             ["pip", "wheel", ".", "--wheel-dir", str(target_path / "wheel")],
@@ -109,7 +110,7 @@ def test_pack_unpack_wheel(tmp_path):
 
     # unpack and assert the file list
     package_name = wheel_name.split("-")[0]
-    unpack_wheel(package_name, wheel_path)
+    await unpack_wheel(package_name, wheel_path)
     _assert_file_list_same(
         path.abspath("convert_prepare/tests/data/pack"),
         tmp_path / "wheel" / "foo-0.1",

py2docfx/convert_prepare/tests/test_sphinx_caller.py

@@ -24,11 +24,11 @@ def init_paths(tmp_path):
     package_info.name = 'testcode'
     return rst_path, destination
 
-
-def test_run_apidoc(tmp_path):
+@pytest.mark.asyncio
+async def test_run_apidoc(tmp_path):
     rst_path, source_code_path = init_paths(tmp_path)
     package_name = "testcode"
-    run_apidoc(package_name, rst_path, source_code_path, package_info.get_exluded_command(), package_info)
+    await run_apidoc(package_name, rst_path, source_code_path, package_info.get_exluded_command(), package_info)
 
     # List all files under rst_path
     rst_list = os.listdir(rst_path)
@@ -39,7 +39,7 @@ def test_run_apidoc(tmp_path):
 async def test_run_converter(tmp_path):
     rst_path, source_code_path = init_paths(tmp_path)
     package_name = "testcode"
-    run_apidoc(package_name, rst_path, source_code_path, package_info.get_exluded_command(), package_info)
+    await run_apidoc(package_name, rst_path, source_code_path, package_info.get_exluded_command(), package_info)
 
     # prepare conf.py, index.rst and docfx_yaml
     conf_path = os.path.abspath("convert_prepare/tests/data/sphinx_caller/conf.py")

py2docfx/docfx_yaml/logger.py

@@ -65,34 +65,6 @@ def get_package_logger(logger_name:str, package_name:str = None):
 
     return file_logger
 
-def log_subprocess_ouput(subprocess_out: subprocess.CompletedProcess, logger: logging.Logger):
-    if subprocess_out.stdout:
-        logger.info(subprocess_out.stdout)
-    if subprocess_out.stderr:
-        msgs = subprocess_out.stderr.split('\n')
-        for msg in msgs:
-            if msg is None or msg == "":
-                continue
-            logger.warning(msg)
-    if subprocess_out.returncode != 0:
-        msg = f"Subprocess failed with return code {subprocess_out.returncode}"
-        logger.error(msg)
-        raise RuntimeError(msg)
-
-def log_git_clone_subprocess_ouput(subprocess_out: subprocess.CompletedProcess, logger: logging.Logger):
-    if subprocess_out.stdout:
-        logger.info(subprocess_out.stdout)
-    if subprocess_out.stderr:
-        msgs = subprocess_out.stderr.split('\n')
-        for msg in msgs:
-            if msg is None or msg == "":
-                continue
-            logger.info(msg)
-    if subprocess_out.returncode != 0:
-        msg = f"Subprocess failed with return code {subprocess_out.returncode}"
-        logger.error(msg)
-        raise RuntimeError(msg)
-
 def counts_errors_warnings(log_file_path):
     error_count = 0
     warning_count = 0
@@ -156,20 +128,53 @@ def output_log_by_log_level():
         log_file_path = os.path.join(package_logs_folder, log_file)
         print_out_log_by_log_level(parse_log(log_file_path), log_level)
 
-async def run_async_subprocess(exe_path, cmd):
-    process = await asyncio.create_subprocess_exec(
-        exe_path, *cmd,
-        stdout=asyncio.subprocess.PIPE,
-        stderr=asyncio.subprocess.PIPE
-    )
+async def run_async_subprocess(exe_path, cmd, logger, cwd=None):
+    if cwd is None:
+        process = await asyncio.create_subprocess_exec(
+            exe_path, *cmd,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE
+        )
+    else:
+        process = await asyncio.create_subprocess_exec(
+            exe_path, *cmd,
+            cwd=cwd,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+    stdout, stderr = await process.communicate()
+    if process.returncode != 0:
+        msg = stderr.decode('utf-8')
+        if msg != None and msg != "":
+            logger.error(msg)
+        raise subprocess.CalledProcessError(process.returncode, cmd, stdout, stderr)
+    else:
+        msg = stdout.decode('utf-8')
+        if msg != None and msg != "":
+            logger.info(msg)
+
+async def run_async_subprocess_without_executable(cmd, logger, cwd=None):
+    if cwd is None:
+        process = await asyncio.create_subprocess_exec(
+            *cmd,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE
+        )
+    else:
+        process = await asyncio.create_subprocess_exec(
+            *cmd,
+            cwd=cwd,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+
     stdout, stderr = await process.communicate()
-    py2docfx_logger = get_logger(__name__)
     if process.returncode != 0:
         msg = stderr.decode('utf-8')
         if msg != None and msg != "":
-            py2docfx_logger.error(msg)
-        raise RuntimeError(msg)
+            logger.error(msg)
+        raise subprocess.CalledProcessError(process.returncode, cmd, stdout, stderr)
     else:
         msg = stdout.decode('utf-8')
         if msg != None and msg != "":
-            py2docfx_logger.info(msg)
+            logger.info(msg)

{py2docfx-0.1.11.dev1994698.dist-info → py2docfx-0.1.11.dev1994975.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: py2docfx
-Version: 0.1.11.dev1994698
+Version: 0.1.11.dev1994975
 Summary: A package built based on Sphinx which download source code package and generate yaml files supported by docfx.
 Author: Microsoft Corporation
 License: MIT License

{py2docfx-0.1.11.dev1994698.dist-info → py2docfx-0.1.11.dev1994975.dist-info}/RECORD

@@ -3,21 +3,21 @@ py2docfx/__main__.py,sha256=TYVpOAmi2MeAg2-d88RxyhCMSxxT6FfCafP_qnzSDKo,5875
 py2docfx/convert_prepare/__init__.py,sha256=XxtxrP0kmW3ZBHIAoxsPDEHzcgeC0WSnole8Lk6CjKs,11
 py2docfx/convert_prepare/arg_parser.py,sha256=Wa1iK8a0Gb3QqaS4X7nUPGvh-7mWhuBS9IVbmnjC770,7606
 py2docfx/convert_prepare/constants.py,sha256=RC5DqNkqWvx4hb91FrajZ1R9dBFLxcPyoEJ43jdm36E,102
-py2docfx/convert_prepare/environment.py,sha256=SpXy3oE9QuDOXK5J3lJd3kl-oXGATUDxe0K137zngXY,6988
+py2docfx/convert_prepare/environment.py,sha256=ClUnNHehYjxMwS50cqnCQO5vHCQaA289opSdfhkOARc,7293
 py2docfx/convert_prepare/generate_conf.py,sha256=wqs6iyElzJarH-20_qEL9zvZvt5xfBMsGXSXPSZy6wg,2295
-py2docfx/convert_prepare/generate_document.py,sha256=vZ0WCHPQfQ11wFLclcKW-7mCkEgvJ1z90yn8BF04u5c,2937
-py2docfx/convert_prepare/get_source.py,sha256=I0ocdVHJ1nMxw8oYeo7BovL2Rg0PCVDpdY62wCK576Q,5244
-py2docfx/convert_prepare/git.py,sha256=0gzh6slKTM9YG0G43m6WAI9zf2SYrWeeMtCxYvJMkmw,6620
-py2docfx/convert_prepare/install_package.py,sha256=hATmgazcSX7k2n4jQXh9sQMyNUc1k1YqHv5K5UMALq4,262
-py2docfx/convert_prepare/pack.py,sha256=0VN7mr-O0oGRb1auGN9FNPDXLY-73q-HLP9bLfmQVIw,1670
+py2docfx/convert_prepare/generate_document.py,sha256=iTOCMBwkfvZEj5dlEGUq4qynUodaYxJuIrG9R5eGk38,2949
+py2docfx/convert_prepare/get_source.py,sha256=Sw5QMWWbKLSznHgNu7JLSW0Wy_FEk0uR_IaKKSKUCl4,5280
+py2docfx/convert_prepare/git.py,sha256=Qb6uexUo3KKwp2Hq9Q27gMU4Z2P8ZBk7JcSmxBfFCBY,6491
+py2docfx/convert_prepare/install_package.py,sha256=aJxQBwLRPTIKpdtLVl-SaXPaF_OX_H1ZtWmcKD8Zzuk,274
+py2docfx/convert_prepare/pack.py,sha256=Jjj9ps8ESoKUFmSk6VgNmxOwMhuwirnQ-rhqigH-4VY,1578
 py2docfx/convert_prepare/package_info.py,sha256=-zrMNeAkHxxzLRjBl1kRehUJy_ugc16yns-xdcAHQIQ,7711
 py2docfx/convert_prepare/package_info_extra_settings.py,sha256=u5B5e8hc0m9PA_-0kJzq1LtKn-xzZlucwXHTFy49mDg,1475
 py2docfx/convert_prepare/params.py,sha256=PXMB8pLtb4XbfI322avA47q0AO-TyBE6kZf7FU8I6v4,1771
 py2docfx/convert_prepare/paths.py,sha256=964RX81Qf__rzXgEATfqBNFCKTYVjLt9J7WCz2TnNdc,485
-py2docfx/convert_prepare/pip_utils.py,sha256=GPWGUBcxW8JKreGwxZIAe9AWAI78aL1aFan8DvaAf1o,2228
+py2docfx/convert_prepare/pip_utils.py,sha256=Y_xmIE0Ld-gXgR8rkXSfj-AMh1bcUJvscMgSdNZyPQU,2138
 py2docfx/convert_prepare/repo_info.py,sha256=6ASJlhBwf6vZTSENgrWCVlJjlJVhuBxzdQyWEdWAC4c,117
 py2docfx/convert_prepare/source.py,sha256=6-A7oof3-WAQcQZZVpT9pKiFLH4CCIZeYqq0MN0O3gw,1710
-py2docfx/convert_prepare/sphinx_caller.py,sha256=axFws3Aqv1hKjzYUxwIaZHs6Ov8z5fvPDQ5Ip5p1jTk,4726
+py2docfx/convert_prepare/sphinx_caller.py,sha256=K7HZddkzENu_QkOrgoKeILsXkRmyQrpUXhRsow-51I4,4696
 py2docfx/convert_prepare/subpackage.py,sha256=mXAi_07pXvnPkSLZfykDh_7VeFxfLy74pYlzhMO8N_Q,5183
 py2docfx/convert_prepare/utils.py,sha256=TwfDjIqIx_nAoSeQ-LX30h_-d96WhPQcAu7P8N_tSjM,1618
 py2docfx/convert_prepare/conf_templates/conf.py_t,sha256=8zxvY1WiG-z2aiSNDY0719C08QxZLXXEMwKfYSGN0ZE,3811
@@ -28,13 +28,13 @@ py2docfx/convert_prepare/subpackage_merge/merge_toc.py,sha256=nkVqe8R0m8D6cyTYV7
 py2docfx/convert_prepare/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 py2docfx/convert_prepare/tests/test_environment.py,sha256=UfeTlfdZKLFcUrSNODw0mvkzZSdbQHBUoK56bdm1xz8,2065
 py2docfx/convert_prepare/tests/test_generate_document.py,sha256=mY8DRT-WRGIBFdDRdFwa7ZSUQwR-fTQtyKwzrzFZLU8,2621
-py2docfx/convert_prepare/tests/test_get_source.py,sha256=49dgQrstyfokG4ZCM7tP6d33v2s7eGFpvb2EcR2GJzc,7711
-py2docfx/convert_prepare/tests/test_pack.py,sha256=xP-Khnj1MfPdIYaUDwXdvXNlI-uHQT3ihZ5d9zCZ914,4195
+py2docfx/convert_prepare/tests/test_get_source.py,sha256=IGBLAemm5SWUnNJJh6GJE5gBtAAdlC_cxS1xSeugzBI,7949
+py2docfx/convert_prepare/tests/test_pack.py,sha256=Gw-LkIcbQuQHpoSbvkybYt4_fVw-LL12dceWw5AUKdE,4242
 py2docfx/convert_prepare/tests/test_package_info.py,sha256=3B-IzmUjETVO-5s3g3Lmh2E6JgopwnRauv8mB-SDZEM,3361
 py2docfx/convert_prepare/tests/test_params.py,sha256=itwmVdBMtU1qIXAGaIoaDfvTOYyAL2B_WLsaBV9KUZY,2232
 py2docfx/convert_prepare/tests/test_post_process_merge_toc.py,sha256=YKOcn4_lf4syGsAvJ9BqpdUUc3SLfK4TiOX1lpXJT_Y,885
 py2docfx/convert_prepare/tests/test_source.py,sha256=LNFZtvjz6QhVLOxatjWokYCCcoSm0bhTikMF9KoTPIE,2025
-py2docfx/convert_prepare/tests/test_sphinx_caller.py,sha256=yi2M4sct4o6-EBKz4Ye9_lyqwYMiSMLNiPhoBfkX5iw,2748
+py2docfx/convert_prepare/tests/test_sphinx_caller.py,sha256=btA6O9IjDSzIecvbjXWvB6r-zhs7rAtKI-eDVOHHOdI,2786
 py2docfx/convert_prepare/tests/test_subpackage.py,sha256=famt8LqwS8hCujtzvRLE5Ih8pa7t45y0d1EBk3UdYns,4984
 py2docfx/convert_prepare/tests/data/generate_document/azure-dummy-sourcecode/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 py2docfx/convert_prepare/tests/data/generate_document/azure-dummy-sourcecode/azure/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -65,7 +65,7 @@ py2docfx/docfx_yaml/convert_enum.py,sha256=HX6qdjDbdbBblx-TH41JXBLIJY3wzL3if-qnL
 py2docfx/docfx_yaml/convert_module.py,sha256=GptO1MRwaQ2Qbu724F0kCDDQQTZe7mWOtrOp3Rzgl-I,2259
 py2docfx/docfx_yaml/convert_package.py,sha256=Ep7PmvoLInDvY6OU5dveR6iVwyzGRkW3q6lX7yGJ0JE,2109
 py2docfx/docfx_yaml/directives.py,sha256=zVVuNM_6AU9G6sbqL1UAyHHgPe7bkBWbthXI-PO5ez0,879
-py2docfx/docfx_yaml/logger.py,sha256=AL0Q_iwheegOPdxBBiiwmraLV6aXLZW3s6Q9d9npAU8,6483
+py2docfx/docfx_yaml/logger.py,sha256=gz416vqSqmsD91qLOCLAzomczlCfWqGXFJFRzLlVbJE,6503
 py2docfx/docfx_yaml/miss_reference.py,sha256=Btoj9wAvA4u_wU7JHH0Cei3910N8a7MS34OUqJvXAd4,2443
 py2docfx/docfx_yaml/nodes.py,sha256=tBDi35jLJArlobl07DKOkmH2qz7dudXLp_kTUfR_r2w,412
 py2docfx/docfx_yaml/parameter_utils.py,sha256=zGSIQrUfbXf9PUK-W_1K83Uo5Zk797Zlze6aMurbHIA,8706
@@ -4193,7 +4193,7 @@ py2docfx/venv/venv1/Lib/site-packages/win32comext/taskscheduler/test/test_addtas
 py2docfx/venv/venv1/Lib/site-packages/win32comext/taskscheduler/test/test_localsystem.py,sha256=08ojAS48W6RLsUbRD45j0SJhg_Y2NFHZT6qjT4Vrig0,75
 py2docfx/venv/venv1/Scripts/pywin32_postinstall.py,sha256=u95n7QQUxpCjrZistYE-3gN451zXzopuJna8cXRQ4Jw,28115
 py2docfx/venv/venv1/Scripts/pywin32_testall.py,sha256=-6yvZmd2lPQc4e8i6PgLsr_totF6mScvoq0Jqr0V2fM,3844
-py2docfx-0.1.11.dev1994698.dist-info/METADATA,sha256=y9yeKqsfN1eAUcFC9C3BtDhWlRl2ypW6d1GiCgV2-20,601
-py2docfx-0.1.11.dev1994698.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-py2docfx-0.1.11.dev1994698.dist-info/top_level.txt,sha256=5dH2uP81dczt_qQJ38wiZ-gzoVWasfiJALWRSjdbnYU,9
-py2docfx-0.1.11.dev1994698.dist-info/RECORD,,
+py2docfx-0.1.11.dev1994975.dist-info/METADATA,sha256=1XFib52JkrdFsQzL7VUQAXxxpcRK7O33jJTA7WiPAOA,601
+py2docfx-0.1.11.dev1994975.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+py2docfx-0.1.11.dev1994975.dist-info/top_level.txt,sha256=5dH2uP81dczt_qQJ38wiZ-gzoVWasfiJALWRSjdbnYU,9
+py2docfx-0.1.11.dev1994975.dist-info/RECORD,,