py2docfx 0.1.10.dev1859594__py3-none-any.whl → 0.1.10.dev1870645__py3-none-any.whl

py2docfx/__main__.py

@@ -1,7 +1,11 @@
 from __future__ import annotations # Avoid A | B annotation break under <= py3.9
+import asyncio
 import argparse
 import os
+import subprocess
 import sys
+import time
+import shutil
 from py2docfx import PACKAGE_ROOT
 from py2docfx.convert_prepare.generate_document import generate_document
 from py2docfx.convert_prepare.get_source import get_source, YAML_OUTPUT_ROOT
@@ -9,6 +13,7 @@ from py2docfx.convert_prepare.install_package import install_package
 from py2docfx.convert_prepare.post_process.merge_toc import merge_toc, move_root_toc_to_target
 from py2docfx.convert_prepare.params import load_file_params, load_command_params
 from py2docfx.convert_prepare.package_info import PackageInfo
+import py2docfx.convert_prepare.environment as py2docfxEnvironment
 
 print("Adding yaml extension to path")
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)),'docfx_yaml'))
@@ -161,7 +166,7 @@ def parse_command_line_args(argv) -> (
 
     if args.param_file_path:
         (package_info_list, required_packages) = load_file_params(args.param_file_path)
-        return (package_info_list, required_packages, github_token,
+        return (list(package_info_list), list(required_packages), github_token,
                 ado_token, output_root, venv_required)
     elif args.param_json:
         (package_info_list, required_packages) = load_command_params(args.param_json)
@@ -213,29 +218,99 @@ def parse_command_line_args(argv) -> (
                 )
         return ([package_info], [], github_token, ado_token, output_root, venv_required)
 
-def install_required_packages(
-        required_package_list: list[PackageInfo], github_token: str, ado_token: str):
-    for idx, package in enumerate(required_package_list):
-        if package.install_type == package.InstallType.SOURCE_CODE:
-            get_source(package, idx, vststoken=ado_token, githubtoken=github_token)
-        install_package(package)
+import py2docfx.convert_prepare.pip_utils as pip_utils
+
+
+# async def create_venv_async(venv_dir: os.PathLike, package: PackageInfo) -> None:
+#     subprocess.run([sys.executable, "-m", "venv", venv_dir], check=True)
+#     venv_exe = os.path.join(venv_dir, "Scripts", "python.exe")
+#     subprocess.run([venv_exe, "-m", "pip", "install", "sphinx==6.1.3"], check=True)
+#     subprocess.run([venv_exe, "-m", "pip", "install", "setuptools"], check=True)
+#     subprocess.run([venv_exe, "-m", "pip", "install", "pyyaml"], check=True)
+#     subprocess.run([venv_exe, "-m", "pip", "install", "jinja2==3.0.3"], check=True)
+#     if package.version:
+#         pkg_install_str = package.name + "==" + package.version
+#     else:
+#         pkg_install_str = package.name
+#     subprocess.run([venv_exe, "-m", "pip", "install", pkg_install_str], check=True)
+#     return venv_exe
+
+from py2docfx.convert_prepare.environment import install_converter_requirements
 
-def donwload_package_generate_documents(
+async def donwload_package_generate_documents(
         package_info_list: list[PackageInfo],
         output_root: str | os.PathLike | None,
         output_doc_folder: os.PathLike | None,
-        github_token: str, ado_token: str, start_num: int):
+        github_token: str, ado_token: str, required_package_list: list):
+    
+    start_num = len(required_package_list)
+    env_prepare_tasks = []
 
+    for idx in range(min([py2docfxEnvironment.VENV_BUFFER, len(package_info_list)])):
+        env_prepare_tasks.append(asyncio.create_task(
+            py2docfxEnvironment.prepare_venv(idx, required_package_list, github_token, ado_token)))
+    
     for idx, package in enumerate(package_info_list):
         package_number = start_num + idx
+        print(f"Processing package {package.name}, env_prepare_tasks: {len(env_prepare_tasks)}")
+        try:
+            await env_prepare_tasks[idx]
+        except Exception as e:
+            print(f"Failed to setup venv for package {package.name}: {e}")
+            raise
+
+        start_time = time.time()
         get_source(package, package_number, vststoken=ado_token, githubtoken=github_token)
-        install_package(package)
-        generate_document(package, output_root)
+        end_time = time.time()
+        print(f"<get_source>{package.name},{end_time-start_time}<get_source/>")
+
+        start_time = time.time()
+        package_name, options = package.get_install_command()
+        try:
+            pip_utils.install_in_exe(py2docfxEnvironment.get_venv_exe(idx), package_name, options)
+        except Exception as e:
+            print(f"Install package failed: {e}")
+            raise
+
+        end_time = time.time()
+        print(f"<install_package>{package.name},{end_time-start_time}<install_package/>")
+
+        # venv_dir = os.path.join(PACKAGE_ROOT, "venv", "0")
+        # venv_exe = asyncio.run(create_venv_async(venv_dir, package))
+
+        start_time = time.time()
+        generate_document(package, output_root, py2docfxEnvironment.get_venv_exe(idx))
+        end_time = time.time()
+        print(f"<generate_document>{package.name},{end_time-start_time}<generate_document/>")
+
+        start_time = time.time()
         merge_toc(YAML_OUTPUT_ROOT, package.path.yaml_output_folder)
+        end_time = time.time()
+        print(f"<merge_toc>{package.name},{end_time-start_time}<merge_toc/>")
+
         if output_doc_folder:
+            start_time = time.time()
             package.path.move_document_to_target(os.path.join(output_doc_folder, package.name))
+            end_time = time.time()
+            print(f"<move_document_to_target>{package.name},{end_time-start_time}<move_document_to_target/>")
+        
+        if idx + py2docfxEnvironment.VENV_BUFFER < len(package_info_list):
+            print(f"Creating venv {idx + py2docfxEnvironment.VENV_BUFFER}")
+            env_prepare_tasks.append(
+                asyncio.create_task(
+                    py2docfxEnvironment.prepare_venv(
+                        idx + py2docfxEnvironment.VENV_BUFFER, 
+                        required_package_list, 
+                        github_token, 
+                        ado_token)
+                )
+            )
+    
     if output_doc_folder:
+        start_time = time.time()
         move_root_toc_to_target(YAML_OUTPUT_ROOT, output_doc_folder)
+        end_time = time.time()
+        print(f"<move_root_toc_to_target>{end_time-start_time}<move_root_toc_to_target/>")
 
 def prepare_out_dir(output_root: str | os.PathLike) -> os.PathLike | None:
     # prepare output_root\DOC_FOLDER_NAME (if folder contains files, raise exception)
@@ -256,16 +331,38 @@ def prepare_out_dir(output_root: str | os.PathLike) -> os.PathLike | None:
         return None
 
 def main(argv) -> int:
+    # clean up TODO: limit those folders to only use py2docfx install position and use a method to iterate
+    if os.path.exists(py2docfxEnvironment.VENV_DIR):
+        shutil.rmtree(py2docfxEnvironment.VENV_DIR)
+    if os.path.exists("dist_temp"):
+        shutil.rmtree("dist_temp")
+    if os.path.exists("source_repo"):
+        shutil.rmtree("source_repo")
+    if os.path.exists("target_repo"):
+        shutil.rmtree("target_repo")
     (package_info_list, required_package_list, github_token, ado_token, output_root, \
           venv_required) = parse_command_line_args(argv)
+    venv_required = False
     if venv_required and sys.prefix == sys.base_prefix:
         raise ValueError("""Please run in a virtual env to prevent breaking your dev
                         environment when not running on pipeliens""")
     output_doc_folder = prepare_out_dir(output_root)
-    install_required_packages(required_package_list, github_token, ado_token)
-    donwload_package_generate_documents(
-        package_info_list, output_root, output_doc_folder,
-        github_token, ado_token, len(list(required_package_list)))
+
+    # start_time = time.time()
+    # install_required_packages(required_package_list, github_token, ado_token)
+    # end_time = time.time()
+    # print(f"<install_required_packages>{end_time-start_time}<install_required_packages/>")
+
+    start_time = time.time()
+    try:
+        asyncio.run(donwload_package_generate_documents(
+            package_info_list, output_root, output_doc_folder,
+            github_token, ado_token, required_package_list))
+    except Exception as e:
+        print(f"An error occurred: {e}")
+        raise
+    end_time = time.time()
+    print(f"<donwload_package_generate_documents>{end_time-start_time}<donwload_package_generate_documents/>")
     return 0
 
 if __name__ == "__main__":

py2docfx/convert_prepare/environment.py

@@ -1,15 +1,23 @@
 import subprocess
+import asyncio
+import os
+from py2docfx import PACKAGE_ROOT
+from py2docfx.convert_prepare.package_info import PackageInfo
+from py2docfx.convert_prepare.get_source import get_source
+from py2docfx.convert_prepare import pip_utils
 
-REQUIREMENT_MODULES = ["setuptools", "sphinx==6.1.3", "pyyaml", "jinja2==3.0.3"]
+REQUIREMENT_MODULES = ["setuptools", "sphinx==6.1.3", "pyyaml", "jinja2==3.0.3", "wheel"]
+VENV_DIR = "venv"
+VENV_BUFFER = 3
+PIP_INSTALL_COMMAND = ["-m", "pip", "install", "--upgrade"]
 
-
-def install_converter_requirements():
+def install_converter_requirements(executable: str):
     """
     Install setuptools/sphinx/pyyaml/jinja2
     Replacing logic of
     https://apidrop.visualstudio.com/Content%20CI/_git/ReferenceAutomation?path=/Python/InstallPackage.ps1&line=15&lineEnd=35&lineStartColumn=1&lineEndColumn=87&lineStyle=plain&_a=contents
     """
-    pip_install_cmd = ["python", "-m", "pip", "install", "--upgrade"]
+    pip_install_cmd = [executable, "-m", "pip", "install", "--upgrade"]
 
     pip_install_common_options = [
         "--no-cache-dir",
@@ -24,3 +32,35 @@ def install_converter_requirements():
         subprocess.run(
             pip_install_cmd + [module] + pip_install_common_options, check=True
         )
+
+def get_venv_path(venv_num: int) -> str:
+    return os.path.join(PACKAGE_ROOT, VENV_DIR, "venv"+str(venv_num))
+
+def get_venv_exe(venv_num: int) -> str:
+    return os.path.join(get_venv_path(venv_num), "Scripts", "python.exe")
+
+async def install_converter_requirement_async(executable: str):
+    pip_cmd = PIP_INSTALL_COMMAND + REQUIREMENT_MODULES + pip_utils.pip_install_common_options
+    await(await asyncio.create_subprocess_exec(executable, *pip_cmd)).wait()
+
+async def install_required_packages(
+        executable: str, required_package_list: list[PackageInfo], github_token: str, ado_token: str):
+    for idx, package in enumerate(required_package_list): # TODO: remove idx
+        # TODO: create a git clone async function and use it here, check if it already exists
+        # if package.install_type == package.InstallType.SOURCE_CODE:
+        #     get_source(package, idx, vststoken=ado_token, githubtoken=github_token)
+        package_name, options = package.get_install_command()
+        pip_cmd = PIP_INSTALL_COMMAND + pip_utils.pip_install_common_options + options + [package_name]
+        await(await asyncio.create_subprocess_exec(executable, *pip_cmd)).wait()
+
+async def create_environment(venv_num: int):
+    await (await asyncio.create_subprocess_exec("python", "-m", "venv", get_venv_path(venv_num))).wait()
+
+async def prepare_venv(venv_num: int, required_package_list: list[PackageInfo], github_token: str, ado_token: str):
+    print(f"<CI INFO>: Creating venv{venv_num}...")
+    await create_environment(venv_num)
+    print(f"<CI INFO>: Installing converter requirements in venv{venv_num}...")
+    await install_converter_requirement_async(get_venv_exe(venv_num))
+    print(f"<CI INFO>: Installing required packages in venv{venv_num}...")
+    await install_required_packages(get_venv_exe(venv_num), required_package_list, github_token, ado_token)
+    print(f"<CI INFO>: venv{venv_num} setup complete.")

py2docfx/convert_prepare/generate_document.py

@@ -1,5 +1,8 @@
 from __future__ import annotations # Avoid A | B annotation break under <= py3.9
 import os
+import sys
+import time
+
 from py2docfx.convert_prepare.generate_conf import generate_conf
 from py2docfx.convert_prepare.git import checkout
 from py2docfx.convert_prepare.package_info import PackageInfo
@@ -8,7 +11,8 @@ from py2docfx.convert_prepare.subpackage import merge_subpackage_files
 
 CONF_TEMPLATE_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "conf_templates")
 
-def generate_document(pkg: PackageInfo, output_root: str | os.PathLike):
+def generate_document(pkg: PackageInfo, output_root: str | os.PathLike, executable=sys.executable):
+    start_time = time.time()
     # Copy manual written RST from target doc repo
     package_paths = pkg.path
     if output_root:
@@ -34,17 +38,27 @@ def generate_document(pkg: PackageInfo, output_root: str | os.PathLike):
     print("<CI INFO>: Listing RST files:")
     for rst_file in os.listdir(package_paths.doc_folder):
         print(rst_file)
-
     print("<CI INFO>: Running Sphinx build...")
+    end_time = time.time()
+    print(f"<pre_generate_conf>{pkg.name},{end_time-start_time}<pre_generate_conf/>")
+    start_time = time.time()
     generate_conf(pkg, package_paths.doc_folder, CONF_TEMPLATE_DIR)
-    run_converter(package_paths.doc_folder, package_paths.yaml_output_folder)
-
+    end_time = time.time()
+    print(f"<generate_conf>{pkg.name},{end_time-start_time}<generate_conf/>")
+    
+    start_time = time.time()
+    run_converter(package_paths.doc_folder, package_paths.yaml_output_folder, executable=executable)
+    end_time = time.time()
+    print(f"<run_converter>{pkg.name},{end_time-start_time}<run_converter/>")
     subpackages_path_record = {}
     if pkg.build_in_subpackage:
+        start_time = time.time()
         subpackages_yaml_path = os.path.join(package_paths.yaml_output_folder, "subpackages")
         for (subpackage_name, subpackage_path) in subpackages_rst_record.items():
             subpackage_yaml_path = os.path.join(subpackages_yaml_path, subpackage_name)
             subpackages_path_record[subpackage_name] = subpackage_yaml_path
-            run_converter(subpackage_path, subpackage_yaml_path)
+            run_converter(subpackage_path, subpackage_yaml_path, executable=executable)
 
         merge_subpackage_files(subpackages_path_record, package_paths.yaml_output_folder, pkg.name)
+        end_time = time.time()
+        print(f"<merge_subpackage_files>{pkg.name},{end_time-start_time}<merge_subpackage_files/>")

py2docfx/convert_prepare/get_source.py

@@ -2,7 +2,7 @@ import os.path as path
 import os
 import subprocess
 import sys
-
+import time
 import py2docfx.convert_prepare.git as git
 import py2docfx.convert_prepare.pip_utils as pip_utils
 import py2docfx.convert_prepare.pack as pack
@@ -75,6 +75,7 @@ def get_source(pkg: PackageInfo, cnt: int, vststoken=None, githubtoken=None):
     dist_dir = path.join("dist_temp", path_cnt)
 
     if pkg.install_type == PackageInfo.InstallType.SOURCE_CODE:
+        start_time = time.time()
         if pkg.url:
             repo_folder = path.join("source_repo", path_cnt)
             token = githubtoken if "github.com" in pkg.url else vststoken
@@ -89,16 +90,20 @@ def get_source(pkg: PackageInfo, cnt: int, vststoken=None, githubtoken=None):
         else:
             source_folder = pkg.folder
             sys.path.insert(0, source_folder)
-
+        end_time = time.time()
+        print(f"<download_source>{pkg.name},{end_time-start_time}<download_source/>")
     elif pkg.install_type == PackageInfo.InstallType.PYPI:
         full_name = pkg.get_combined_name_version()
+        start_time = time.time()
         pip_utils.download(
             full_name,
             dist_dir,
             extra_index_url=pkg.extra_index_url,
             prefer_source_distribution=pkg.prefer_source_distribution,
         )
-
+        end_time = time.time()
+        print(f"<download_pypi>{pkg.name},{end_time-start_time}<download_pypi/>")
+        start_time = time.time()
         # unpack the downloaded wheel file.
         downloaded_dist_file = path.join(dist_dir, os.listdir(dist_dir)[0])
         pack.unpack_dist(downloaded_dist_file)
@@ -107,9 +112,15 @@ def get_source(pkg: PackageInfo, cnt: int, vststoken=None, githubtoken=None):
             path.dirname(downloaded_dist_file),
             os.listdir(dist_dir)[0]
         )
+        end_time = time.time()
+        print(f"<unpack_pypi>{pkg.name},{end_time-start_time}<unpack_pypi/>")
     elif pkg.install_type == PackageInfo.InstallType.DIST_FILE:
+        start_time = time.time()
         pip_utils.download(pkg.location, dist_dir, prefer_source_distribution=False)
+        end_time = time.time()
+        print(f"<download_dist>{pkg.name},{end_time-start_time}<download_dist/>")
 
+        start_time = time.time()
         # unpack the downloaded dist file.
         downloaded_dist_file = path.join(dist_dir, os.listdir(dist_dir)[0])
         pack.unpack_dist(downloaded_dist_file)
@@ -121,7 +132,12 @@ def get_source(pkg: PackageInfo, cnt: int, vststoken=None, githubtoken=None):
             path.dirname(downloaded_dist_file),
             os.listdir(dist_dir)[0]
         )
+        end_time = time.time()
+        print(f"<unpack_dist>{pkg.name},{end_time-start_time}<unpack_dist/>")
     else:
         raise ValueError(f"Unknown install type: {pkg.install_type}")
 
+    start_time = time.time()
     update_package_info(pkg, source_folder)
+    end_time = time.time()
+    print(f"<update_package_info>{pkg.name},{end_time-start_time}<update_package_info/>")

py2docfx/convert_prepare/pip_utils.py

@@ -1,5 +1,7 @@
 import subprocess
+import time
 from py2docfx import PACKAGE_ROOT
+
 PYPI = "pypi"
 
 pip_install_common_options = [
@@ -31,3 +33,11 @@ def install(package_name, options):
         " ".join(pip_install_common_options + options), package_name
     ).split(" ")
     subprocess.run(install_param, check=True, cwd=PACKAGE_ROOT)
+
+
+def install_in_exe(exe_path, package_name, options):
+    # Installs a package from PyPI using pip.
+    install_param = [exe_path] + "-m pip install {} {}".format(
+        " ".join(pip_install_common_options + options), package_name
+    ).split(" ")
+    subprocess.run(install_param, check=True, cwd=PACKAGE_ROOT)

py2docfx/convert_prepare/sphinx_caller.py

@@ -10,7 +10,7 @@ from py2docfx.convert_prepare.package_info import PackageInfo
 from py2docfx.convert_prepare.paths import folder_is_hidden
 from py2docfx.convert_prepare.subpackage import (get_subpackages,
                                         move_rst_files_to_subfolder)
-
+import time
 DEBUG_SPHINX_FLAG = 'PY2DOCFX_DEBUG_SPHINX'
 
 def run_apidoc(rst_path, source_code_path, exclude_paths, package_info: PackageInfo):
@@ -48,7 +48,7 @@ def run_apidoc(rst_path, source_code_path, exclude_paths, package_info: PackageI
     return subpackages_rst_record
 
 
-def run_converter(rst_path, out_path, conf_path = None):
+def run_converter(rst_path, out_path, conf_path = None, executable = sys.executable):
     """
     Take rst files as input and run sphinx converter
 
@@ -78,7 +78,8 @@ def run_converter(rst_path, out_path, conf_path = None):
         if not sys.executable:
             raise ValueError("Can't get the executable binary for the Python interpreter.")
         sphinx_param = [
-            sys.executable, sphinx_build_path,
+            executable,
+            sphinx_build_path,
             rst_path,
             outdir,
             '-c', conf_path or rst_path,
@@ -95,5 +96,8 @@ def run_converter(rst_path, out_path, conf_path = None):
             env_tmp["PYTHONPATH"] = f"{package_root_parent};{python_path_current}" if python_path_current else package_root_parent
         else:
             env_tmp["PYTHONPATH"] = f"{package_root_parent}:{python_path_current}" if python_path_current else package_root_parent
+        start_time = time.time()
         subprocess.run(sphinx_param, check=True, cwd=PACKAGE_ROOT, env=env_tmp)
+        end_time = time.time()
+        print(f"<run_sphinx>{out_path},{end_time-start_time}<run_sphinx/>")
     return outdir

py2docfx/convert_prepare/tests/test_generate_document.py

@@ -51,4 +51,3 @@ def test_generate_document(tmp_path):
     assert not os.path.exists(os.path.join(yaml_path, "azure.dummy.yml"))
     assert not os.path.exists(os.path.join(yaml_path, "azure.yml"))
     assert os.path.exists(os.path.join(yaml_path, "toc.yml"))
-

py2docfx/docfx_yaml/tests/roots/test-writer-uri/code_with_uri.py

@@ -17,12 +17,5 @@ class SampleClass():
         """
         pass
 
-    def dummy_summary3(self):
-        """
-        This is a bare URL that shouldn't be transformed into a link
-        because it's in the exclusion list: https://management.azure.com
-        """
-        pass
-
     pass
     

py2docfx/docfx_yaml/tests/test_writer_uri.py

@@ -20,7 +20,6 @@ def test_http_link_in_summary_should_not_nest_parenthesis(app):
     class_summary_result = transform_node(app, doctree[1][1][0])
     method1_summary_result = transform_node(app, doctree[1][1][2][1])
     method2_summary_result = transform_node(app, doctree[1][1][4][1])
-    method3_summary_result = transform_node(app, doctree[1][1][6][1])
 
     # Assert
     # Shouldn't see something like [title]((link))
@@ -29,10 +28,7 @@ def test_http_link_in_summary_should_not_nest_parenthesis(app):
                                "We should not generate nested parenthesis causing docs validation warnings\n")
     method2_summary_expected = ("\n\n   This isn't a content issue link ([https://www.microsoft.com](https://www.microsoft.com))\n   "
                                 "Should expect a transformed Markdown link.\n")
-    method3_summary_expected = ("\n\n   This is a bare URL that shouldn't be transformed into a link\n   "
-                                "because it's in the exclusion list: https://management.azure.com\n")
     assert(class_summary_expected == class_summary_result)
     assert(method1_summary_expected == method1_summary_result)
     assert(method2_summary_expected == method2_summary_result)
-    assert(method3_summary_expected == method3_summary_result)
     

py2docfx/docfx_yaml/writer.py

@@ -183,15 +183,6 @@ class MarkdownTranslator(nodes.NodeVisitor):
     sectionchars = '*=-~"+`'
     xref_template = "<xref:{0}>"
 
-    # URLs that shouldn't be automatically rendered as hyperlinks if found bare. Included because they appear
-    # frequently, get flagged by the broken link validator and/or there's no value to the user in
-    # making them clickable links.
-    urls_that_shouldnt_be_rendered_as_links = {
-        "https://management.azure.com",
-        "https://management.chinacloudapi.cn",
-        "https://management.usgovcloudapi.net"
-        }
-
     def __init__(self, document, builder):
         self.invdata = []
         nodes.NodeVisitor.__init__(self, document)
@@ -934,10 +925,7 @@ class MarkdownTranslator(nodes.NodeVisitor):
                             match_content_issue_pattern = True
                             ref_string = node.attributes["refuri"]
                 if not match_content_issue_pattern:
-                    if inner_text == node.attributes['refuri'] and inner_text in cls.urls_that_shouldnt_be_rendered_as_links:
-                        ref_string = inner_text
-                    else:
-                        ref_string = '[{}]({})'.format(node.astext(), node.attributes['refuri'])
+                    ref_string = '[{}]({})'.format(node.astext(), node.attributes['refuri'])
             else:
                 # only use id in class and func refuri if its id exists
                 # otherwise, remove '.html#' in refuri