py2docfx 0.1.10.dev1871013__py3-none-any.whl → 0.1.10.dev1871296__py3-none-any.whl

py2docfx/__main__.py

@@ -247,10 +247,13 @@ async def donwload_package_generate_documents(
     env_prepare_tasks = []
     env_remove_tasks = []
 
+    start_time = time.time()
     for idx in range(min([py2docfxEnvironment.VENV_BUFFER, len(package_info_list)])):
         env_prepare_tasks.append(asyncio.create_task(
             py2docfxEnvironment.prepare_venv(idx, required_package_list, github_token, ado_token)))
-    
+    end_time = time.time()
+    print(f"<warmup_venvs>{end_time-start_time}<warmup_venvs/>")
+
     for idx, package in enumerate(package_info_list):
         package_number = start_num + idx
         print(f"Processing package {package.name}, env_prepare_tasks: {len(env_prepare_tasks)}")
@@ -261,10 +264,10 @@ async def donwload_package_generate_documents(
             print(f"Failed to setup venv for package {package.name}: {e}")
             raise
         end_time = time.time()
-        print(f"<wait_prepare_venv>{end_time-start_time}<wait_prepare_venv/>")
+        print(f"<wait_prepare_venv>{package.name},{end_time-start_time}<wait_prepare_venv/>")
 
         start_time = time.time()
-        get_source(package, package_number, vststoken=ado_token, githubtoken=github_token)
+        get_source(py2docfxEnvironment.get_venv_exe(idx), package, package_number, vststoken=ado_token, githubtoken=github_token)
         end_time = time.time()
         print(f"<get_source>{package.name},{end_time-start_time}<get_source/>")
 
@@ -298,6 +301,7 @@ async def donwload_package_generate_documents(
             end_time = time.time()
             print(f"<move_document_to_target>{package.name},{end_time-start_time}<move_document_to_target/>")
         
+        start_time = time.time()
         if idx + py2docfxEnvironment.VENV_BUFFER < len(package_info_list):
             print(f"Creating venv {idx + py2docfxEnvironment.VENV_BUFFER}")
             env_prepare_tasks.append(
@@ -309,11 +313,20 @@ async def donwload_package_generate_documents(
                         ado_token)
                 )
             )
+        end_time = time.time()
+        print(f"<create_prepare_venv>{package.name},{end_time-start_time}<create_prepare_venv/>")
         if idx >= 1:
-            env_remove_tasks.append(asyncio.create_task(py2docfxEnvironment.remove_environment(idx-1)))
+            start_time = time.time()
+            env_remove_tasks.append(asyncio.create_task(
+                py2docfxEnvironment.remove_environment(idx-1)))
+            end_time = time.time()
+            print(f"<create_remove_venv>{package.name},{end_time-start_time}<create_remove_venv/>")
         if idx > py2docfxEnvironment.VENV_BUFFER and env_remove_tasks[idx-py2docfxEnvironment.VENV_BUFFER] != None:
+            start_time = time.time()
             print(f"Removing venv {idx-py2docfxEnvironment.VENV_BUFFER}")
             await env_remove_tasks[idx-py2docfxEnvironment.VENV_BUFFER]
+            end_time = time.time()
+            print(f"<wait_remove_venv>{package.name},{end_time-start_time}<wait_remove_venv/>")
     
     if output_doc_folder:
         start_time = time.time()
@@ -321,9 +334,12 @@ async def donwload_package_generate_documents(
         end_time = time.time()
         print(f"<move_root_toc_to_target>{end_time-start_time}<move_root_toc_to_target/>")
     
+    start_time = time.time()
     for idx in range(len(env_remove_tasks)):
         if env_remove_tasks[idx] != None and not env_remove_tasks[idx].done():
             await env_remove_tasks[idx]
+    end_time = time.time()
+    print(f"<wait_remove_venv>{end_time-start_time}<wait_remove_venv/>")
 
 def prepare_out_dir(output_root: str | os.PathLike) -> os.PathLike | None:
     # prepare output_root\DOC_FOLDER_NAME (if folder contains files, raise exception)

py2docfx/convert_prepare/environment.py

@@ -77,7 +77,14 @@ async def remove_environment(venv_num: int):
     venv_path = get_venv_path(venv_num)
     if os.path.exists(venv_path):
         print(f"<CI INFO>: Removing venv{venv_num}...")
-        return asyncio.to_thread(shutil.rmtree, venv_path)
-
-# Example usage
-# asyncio.run(remove_environment(1))
+        # Create a subprocess to run the shell command for removing the directory
+        process = await(await asyncio.create_subprocess_shell(
+            f'rm -rf {venv_path}' if os.name != 'nt' else f'rmdir /S /Q {venv_path}',
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE
+        )).wait()
+        stdout, stderr = await process.communicate()
+        if process.returncode == 0:
+            print(f"<CI INFO>: venv{venv_num} removed.")
+        else:
+            print(f"<CI ERROR>: Failed to remove venv{venv_num}. Error: {stderr.decode()}")

py2docfx/convert_prepare/get_source.py

@@ -11,7 +11,7 @@ from py2docfx.convert_prepare.source import Source
 
 YAML_OUTPUT_ROOT =  path.join("target_repo", "docs-ref-autogen")
 
-def update_package_info(pkg: PackageInfo, source_folder: str):
+def update_package_info(executable: str, pkg: PackageInfo, source_folder: str):
     cur_path = os.getcwd()
     os.chdir(source_folder) # TODO: replace it
 
@@ -23,7 +23,7 @@ def update_package_info(pkg: PackageInfo, source_folder: str):
     if "setup.py" in files:
         for attr in attrs:
             proc_ret = subprocess.run(
-                ["python", "setup.py", "--quiet", "--dry-run", f"--{attr}"],
+                [executable, "setup.py", "--quiet", "--dry-run", f"--{attr}"],
                 capture_output=True,
                 text=True,
                 check=True
@@ -70,7 +70,7 @@ def update_package_info(pkg: PackageInfo, source_folder: str):
         source_folder=source_folder, yaml_output_folder=yaml_output_folder, package_name=pkg.name
     )
 
-def get_source(pkg: PackageInfo, cnt: int, vststoken=None, githubtoken=None):
+def get_source(executable: str, pkg: PackageInfo, cnt: int, vststoken=None, githubtoken=None):
     path_cnt = str(cnt)
     dist_dir = path.join("dist_temp", path_cnt)
 
@@ -138,6 +138,6 @@ def get_source(pkg: PackageInfo, cnt: int, vststoken=None, githubtoken=None):
         raise ValueError(f"Unknown install type: {pkg.install_type}")
 
     start_time = time.time()
-    update_package_info(pkg, source_folder)
+    update_package_info(executable, pkg, source_folder)
     end_time = time.time()
     print(f"<update_package_info>{pkg.name},{end_time-start_time}<update_package_info/>")

py2docfx/convert_prepare/tests/test_get_source.py

@@ -5,6 +5,7 @@ import os
 import shutil
 import stat
 import pytest
+import sys
 from py2docfx.convert_prepare import git
 from py2docfx.convert_prepare import get_source
 from py2docfx.convert_prepare.package_info import PackageInfo
@@ -44,19 +45,19 @@ def test_update_package_info(init_package_info):
     base_path = "convert_prepare/tests/data/get_source/"
 
     # case of setup.py
-    get_source.update_package_info(package, os.path.join(base_path, "mock-1"))
+    get_source.update_package_info(sys.executable, package, os.path.join(base_path, "mock-1"))
     assert package.name == "dummy_package"
     assert package.version == "3.1.0"
 
     # case of metadata
     package = init_package_info
-    get_source.update_package_info(package, os.path.join(base_path, "mock-2"))
+    get_source.update_package_info(sys.executable, package, os.path.join(base_path, "mock-2"))
     assert package.name == "mock_package"
     assert package.version == "2.2.0"
 
     # case of dist-info folder
     package = init_package_info
-    get_source.update_package_info(package, os.path.join(base_path, "mock-3"))
+    get_source.update_package_info(sys.executable, package, os.path.join(base_path, "mock-3"))
     assert package.name == "mock_package"
     assert package.version == "1.2.0"
 
@@ -75,7 +76,7 @@ def test_get_source_git_clone(init_package_info):
     package.branch = "main"
     package.folder = None
     package.url = "https://github.com/Azure/azure-iot-hub-python"
-    get_source.get_source(package, 0)
+    get_source.get_source(sys.executable, package, 0)
     assert git.status("source_repo/0") is True
 
 def test_get_source_dist_file_zip(init_package_info):
@@ -91,7 +92,7 @@ def test_get_source_dist_file_zip(init_package_info):
     package.extra_index_url = None
     package.prefer_source_distribution = True
     package.location = "https://files.pythonhosted.org/packages/3e/71/f6f71a276e2e69264a97ad39ef850dca0a04fce67b12570730cb38d0ccac/azure-common-1.1.28.zip"
-    get_source.get_source(package, 1)
+    get_source.get_source(sys.executable, package, 1)
     assert os.path.exists("dist_temp/1/azure-common-1.1.28")
     assert package.path.source_folder == "dist_temp\\1\\azure-common-1.1.28"
     assert os.path.exists("dist_temp/1/azure-common-1.1.28.zip") is False
@@ -109,7 +110,7 @@ def test_get_source_dist_file_whl(init_package_info):
     package.extra_index_url = None
     package.prefer_source_distribution = True
     package.location = "https://files.pythonhosted.org/packages/62/55/7f118b9c1b23ec15ca05d15a578d8207aa1706bc6f7c87218efffbbf875d/azure_common-1.1.28-py2.py3-none-any.whl"
-    get_source.get_source(package, 2)
+    get_source.get_source(sys.executable, package, 2)
     assert os.path.exists("dist_temp/2/azure_common-1.1.28")
     assert package.path.source_folder == "dist_temp\\2\\azure_common-1.1.28"
     assert os.path.exists("dist_temp/2/azure_common-1.1.28-py2.py3-none-any.whl") is False
@@ -127,7 +128,7 @@ def test_get_source_dist_file_tar(init_package_info):
     package.extra_index_url = None
     package.prefer_source_distribution = True
     package.location = "https://files.pythonhosted.org/packages/fa/19/43a9eb812b4d6071fdc2c55640318f7eb5a1be8dbd3b6f9d96a1996e1bb6/azure-core-1.29.4.tar.gz"
-    get_source.get_source(package, 3)
+    get_source.get_source(sys.executable, package, 3)
     assert os.path.exists("dist_temp/3/azure-core-1.29.4")
     assert package.path.source_folder == "dist_temp\\3\\azure-core-1.29.4"
     assert os.path.exists("dist_temp/3/azure-core-1.29.4.tar.gz") is False
@@ -144,7 +145,7 @@ def test_get_source_pip_whl(init_package_info):
     package.build_in_subpackage = False
     package.extra_index_url = None
     package.prefer_source_distribution = False
-    get_source.get_source(package, 4)
+    get_source.get_source(sys.executable, package, 4)
     assert os.path.exists("dist_temp/4/azure_common-1.1.28")
     assert package.path.source_folder == "dist_temp\\4\\azure_common-1.1.28"
     assert os.path.exists("dist_temp/4/azure_common-1.1.28-py2.py3-none-any.whl") is False
@@ -161,7 +162,7 @@ def test_get_source_pip_zip(init_package_info):
     package.build_in_subpackage = False
     package.extra_index_url = None
     package.prefer_source_distribution = True
-    get_source.get_source(package, 5)
+    get_source.get_source(sys.executable, package, 5)
     assert os.path.exists("dist_temp/5/azure-common-1.1.28")
     assert package.path.source_folder == "dist_temp\\5\\azure-common-1.1.28"
     assert os.path.exists("dist_temp/5/azure-common-1.1.28.zip") is False
@@ -178,7 +179,7 @@ def test_get_source_zip_file_at_position_0(init_package_info):
     package.build_in_subpackage = False
     package.extra_index_url = None
     package.prefer_source_distribution = True
-    get_source.get_source(package, 6)
+    get_source.get_source(sys.executable, package, 6)
     assert os.path.exists("dist_temp/6/azure_template-0.1.0b3942895")
     assert package.path.source_folder == "dist_temp\\6\\azure_template-0.1.0b3942895"
     assert os.path.exists("dist_temp/6/azure_template-0.1.0b3942895.tar.gz") is False

{py2docfx-0.1.10.dev1871013.dist-info → py2docfx-0.1.10.dev1871296.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: py2docfx
-Version: 0.1.10.dev1871013
+Version: 0.1.10.dev1871296
 Summary: A package built based on Sphinx which download source code package and generate yaml files supported by docfx.
 Author: Microsoft Corporation
 License: MIT License

{py2docfx-0.1.10.dev1871013.dist-info → py2docfx-0.1.10.dev1871296.dist-info}/RECORD

@@ -1,10 +1,10 @@
 py2docfx/__init__.py,sha256=kPRhPGPC1JknDotkksG428c1iIgfFr_4_7Jm-llrowY,72
-py2docfx/__main__.py,sha256=Nq5DzPY8RZNuUqUUTW6rF_k_75LNrGG39lX4injCWrY,15719
+py2docfx/__main__.py,sha256=3vyfQ8XV8TtKPCb7C7A0d-VnzKXb58zfHOz806lUBa0,16554
 py2docfx/convert_prepare/__init__.py,sha256=XxtxrP0kmW3ZBHIAoxsPDEHzcgeC0WSnole8Lk6CjKs,11
-py2docfx/convert_prepare/environment.py,sha256=_gGbLRnuOnL8vsU-b7jZITqIN9CUeSXNFB8_xKWtKjk,3656
+py2docfx/convert_prepare/environment.py,sha256=PDQfOUHEHFIZUiBPDK1zsrOM_dnC0HES-IpClk7a_us,4140
 py2docfx/convert_prepare/generate_conf.py,sha256=wqs6iyElzJarH-20_qEL9zvZvt5xfBMsGXSXPSZy6wg,2295
 py2docfx/convert_prepare/generate_document.py,sha256=cLfFr7od0RHADkXyh2gaPreXZGvKNBUlduBosYTuPbk,3118
-py2docfx/convert_prepare/get_source.py,sha256=eEIbfLkGdBSOIJElMBTiXkKlkujb_hdFu5g6InNroTg,5851
+py2docfx/convert_prepare/get_source.py,sha256=hvWR_yPlfliLI9wywtnimKpjhJkx-LUYD6adGHgYTFA,5899
 py2docfx/convert_prepare/git.py,sha256=xGJp2nDWLfVljrxyPnFKPoLIqmBh6by-QdITogIuxi0,5893
 py2docfx/convert_prepare/install_package.py,sha256=hATmgazcSX7k2n4jQXh9sQMyNUc1k1YqHv5K5UMALq4,262
 py2docfx/convert_prepare/pack.py,sha256=vZS67_GzEhUmZWHU1dxm8gnWyRBs-kB6-KjX1d_FdOU,1260
@@ -24,7 +24,7 @@ py2docfx/convert_prepare/subpackage_merge/merge_root_package.py,sha256=uK96qL2as
 py2docfx/convert_prepare/subpackage_merge/merge_toc.py,sha256=nkVqe8R0m8D6cyTYV7aIpMDXorvn4-LXfU_vIK_hJBg,1706
 py2docfx/convert_prepare/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 py2docfx/convert_prepare/tests/test_generate_document.py,sha256=FpxwyRQFBfEdYUH16tpcJrqTAC0J5mq-Edn-N7Lz0rY,2413
-py2docfx/convert_prepare/tests/test_get_source.py,sha256=70EVlIZe71aDYDOXMnhqzZclKL8KuIg4t-X0U-pVyhA,7429
+py2docfx/convert_prepare/tests/test_get_source.py,sha256=pry20HGWLfQGDtpNoj2uc_y1AwgNGWEvBWPpJUEVSVY,7601
 py2docfx/convert_prepare/tests/test_pack.py,sha256=46JWMNzknIptDVs7D3CuxcmqBr_OKMmaw1br9H7wqco,4134
 py2docfx/convert_prepare/tests/test_package_info.py,sha256=3B-IzmUjETVO-5s3g3Lmh2E6JgopwnRauv8mB-SDZEM,3361
 py2docfx/convert_prepare/tests/test_params.py,sha256=C-aY9xDjsNTcGQATco_J_i4nF4JSmaeFPEMI3-NtiGE,2229
@@ -1882,7 +1882,7 @@ py2docfx/venv/venv0/Scripts/rst2s5.py,sha256=lnv7lRNv1Oz3gjI5BOJlTqfCJQLJxgwJ9l3
 py2docfx/venv/venv0/Scripts/rst2xetex.py,sha256=5_9VjwA5RKUMJG9L__1ZiS8DIeLHRdPWeWW7B4otM8M,909
 py2docfx/venv/venv0/Scripts/rst2xml.py,sha256=hU2nzGVbXbQBnH_ydI5f2VDkGQyDMUdTgz3rfa45JMg,638
 py2docfx/venv/venv0/Scripts/rstpep2html.py,sha256=uLmCJKfEKPeeWS0n76HharN9YEYIuCRQQXfPPBJaRso,706
-py2docfx-0.1.10.dev1871013.dist-info/METADATA,sha256=tnMc1MCAhdOQTwQam94xJuJfhlBL1Elbd28qOYpzGFI,601
-py2docfx-0.1.10.dev1871013.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-py2docfx-0.1.10.dev1871013.dist-info/top_level.txt,sha256=5dH2uP81dczt_qQJ38wiZ-gzoVWasfiJALWRSjdbnYU,9
-py2docfx-0.1.10.dev1871013.dist-info/RECORD,,
+py2docfx-0.1.10.dev1871296.dist-info/METADATA,sha256=bblQMXf6CUjCcgJcRO1yU0S01CIXfig8J_Rpdk6-FbQ,601
+py2docfx-0.1.10.dev1871296.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+py2docfx-0.1.10.dev1871296.dist-info/top_level.txt,sha256=5dH2uP81dczt_qQJ38wiZ-gzoVWasfiJALWRSjdbnYU,9
+py2docfx-0.1.10.dev1871296.dist-info/RECORD,,