py2docfx 0.1.10.dev1870753__py3-none-any.whl → 0.1.10.dev1871193__py3-none-any.whl

py2docfx/__main__.py

@@ -245,6 +245,7 @@ async def donwload_package_generate_documents(
     
     start_num = len(required_package_list)
     env_prepare_tasks = []
+    env_remove_tasks = []
 
     for idx in range(min([py2docfxEnvironment.VENV_BUFFER, len(package_info_list)])):
         env_prepare_tasks.append(asyncio.create_task(
@@ -263,7 +264,7 @@ async def donwload_package_generate_documents(
         print(f"<wait_prepare_venv>{end_time-start_time}<wait_prepare_venv/>")
 
         start_time = time.time()
-        get_source(package, package_number, vststoken=ado_token, githubtoken=github_token)
+        get_source(py2docfxEnvironment.get_venv_exe(idx), package, package_number, vststoken=ado_token, githubtoken=github_token)
         end_time = time.time()
         print(f"<get_source>{package.name},{end_time-start_time}<get_source/>")
 
@@ -308,13 +309,22 @@ async def donwload_package_generate_documents(
                         ado_token)
                 )
             )
-        py2docfxEnvironment.remove_environment(idx)
+        if idx >= 1:
+            env_remove_tasks.append(asyncio.create_task(
+                asyncio.to_thread(py2docfxEnvironment.remove_environment,idx-1)))
+        if idx > py2docfxEnvironment.VENV_BUFFER and env_remove_tasks[idx-py2docfxEnvironment.VENV_BUFFER] != None:
+            print(f"Removing venv {idx-py2docfxEnvironment.VENV_BUFFER}")
+            await env_remove_tasks[idx-py2docfxEnvironment.VENV_BUFFER]
     
     if output_doc_folder:
         start_time = time.time()
         move_root_toc_to_target(YAML_OUTPUT_ROOT, output_doc_folder)
         end_time = time.time()
         print(f"<move_root_toc_to_target>{end_time-start_time}<move_root_toc_to_target/>")
+    
+    for idx in range(len(env_remove_tasks)):
+        if env_remove_tasks[idx] != None and not env_remove_tasks[idx].done():
+            await env_remove_tasks[idx]
 
 def prepare_out_dir(output_root: str | os.PathLike) -> os.PathLike | None:
     # prepare output_root\DOC_FOLDER_NAME (if folder contains files, raise exception)
@@ -344,6 +354,8 @@ def main(argv) -> int:
         shutil.rmtree("source_repo")
     if os.path.exists("target_repo"):
         shutil.rmtree("target_repo")
+
+    # TODO: may need to purge pip cache
     (package_info_list, required_package_list, github_token, ado_token, output_root, \
           venv_required) = parse_command_line_args(argv)
     venv_required = False

py2docfx/convert_prepare/environment.py

@@ -10,6 +10,7 @@ from py2docfx.convert_prepare import pip_utils
 REQUIREMENT_MODULES = ["setuptools", "sphinx==6.1.3", "pyyaml", "jinja2==3.0.3", "wheel"]
 VENV_DIR = "venv"
 VENV_BUFFER = 10
+VENV_DELETE_BUFFER = 10
 PIP_INSTALL_COMMAND = ["-m", "pip", "install", "--upgrade"]
 
 PIP_INSTALL_VENV_COMMON_OPTIONS = [
@@ -73,7 +74,10 @@ async def prepare_venv(venv_num: int, required_package_list: list[PackageInfo],
     print(f"<CI INFO>: venv{venv_num} setup complete.")
 
 def remove_environment(venv_num: int):
-    if os.path.exists(get_venv_path(venv_num)):
+    venv_path = get_venv_path(venv_num)
+    if os.path.exists(venv_path):
         print(f"<CI INFO>: Removing venv{venv_num}...")
-        shutil.rmtree(get_venv_path(venv_num))
-        print(f"<CI INFO>: venv{venv_num} removed.")
+        shutil.rmtree(venv_path)
+
+# Example usage
+# asyncio.run(remove_environment(1))

py2docfx/convert_prepare/get_source.py

@@ -11,7 +11,7 @@ from py2docfx.convert_prepare.source import Source
 
 YAML_OUTPUT_ROOT =  path.join("target_repo", "docs-ref-autogen")
 
-def update_package_info(pkg: PackageInfo, source_folder: str):
+def update_package_info(executable: str, pkg: PackageInfo, source_folder: str):
     cur_path = os.getcwd()
     os.chdir(source_folder) # TODO: replace it
 
@@ -23,7 +23,7 @@ def update_package_info(pkg: PackageInfo, source_folder: str):
     if "setup.py" in files:
         for attr in attrs:
             proc_ret = subprocess.run(
-                ["python", "setup.py", "--quiet", "--dry-run", f"--{attr}"],
+                [executable, "setup.py", "--quiet", "--dry-run", f"--{attr}"],
                 capture_output=True,
                 text=True,
                 check=True
@@ -70,7 +70,7 @@ def update_package_info(pkg: PackageInfo, source_folder: str):
         source_folder=source_folder, yaml_output_folder=yaml_output_folder, package_name=pkg.name
     )
 
-def get_source(pkg: PackageInfo, cnt: int, vststoken=None, githubtoken=None):
+def get_source(executable: str, pkg: PackageInfo, cnt: int, vststoken=None, githubtoken=None):
     path_cnt = str(cnt)
     dist_dir = path.join("dist_temp", path_cnt)
 
@@ -138,6 +138,6 @@ def get_source(pkg: PackageInfo, cnt: int, vststoken=None, githubtoken=None):
         raise ValueError(f"Unknown install type: {pkg.install_type}")
 
     start_time = time.time()
-    update_package_info(pkg, source_folder)
+    update_package_info(executable, pkg, source_folder)
     end_time = time.time()
     print(f"<update_package_info>{pkg.name},{end_time-start_time}<update_package_info/>")

py2docfx/convert_prepare/tests/test_get_source.py

@@ -5,6 +5,7 @@ import os
 import shutil
 import stat
 import pytest
+import sys
 from py2docfx.convert_prepare import git
 from py2docfx.convert_prepare import get_source
 from py2docfx.convert_prepare.package_info import PackageInfo
@@ -44,19 +45,19 @@ def test_update_package_info(init_package_info):
     base_path = "convert_prepare/tests/data/get_source/"
 
     # case of setup.py
-    get_source.update_package_info(package, os.path.join(base_path, "mock-1"))
+    get_source.update_package_info(sys.executable, package, os.path.join(base_path, "mock-1"))
     assert package.name == "dummy_package"
     assert package.version == "3.1.0"
 
     # case of metadata
     package = init_package_info
-    get_source.update_package_info(package, os.path.join(base_path, "mock-2"))
+    get_source.update_package_info(sys.executable, package, os.path.join(base_path, "mock-2"))
     assert package.name == "mock_package"
     assert package.version == "2.2.0"
 
     # case of dist-info folder
     package = init_package_info
-    get_source.update_package_info(package, os.path.join(base_path, "mock-3"))
+    get_source.update_package_info(sys.executable, package, os.path.join(base_path, "mock-3"))
     assert package.name == "mock_package"
     assert package.version == "1.2.0"
 
@@ -75,7 +76,7 @@ def test_get_source_git_clone(init_package_info):
     package.branch = "main"
     package.folder = None
     package.url = "https://github.com/Azure/azure-iot-hub-python"
-    get_source.get_source(package, 0)
+    get_source.get_source(sys.executable, package, 0)
     assert git.status("source_repo/0") is True
 
 def test_get_source_dist_file_zip(init_package_info):
@@ -91,7 +92,7 @@ def test_get_source_dist_file_zip(init_package_info):
     package.extra_index_url = None
     package.prefer_source_distribution = True
     package.location = "https://files.pythonhosted.org/packages/3e/71/f6f71a276e2e69264a97ad39ef850dca0a04fce67b12570730cb38d0ccac/azure-common-1.1.28.zip"
-    get_source.get_source(package, 1)
+    get_source.get_source(sys.executable, package, 1)
     assert os.path.exists("dist_temp/1/azure-common-1.1.28")
     assert package.path.source_folder == "dist_temp\\1\\azure-common-1.1.28"
     assert os.path.exists("dist_temp/1/azure-common-1.1.28.zip") is False
@@ -109,7 +110,7 @@ def test_get_source_dist_file_whl(init_package_info):
     package.extra_index_url = None
     package.prefer_source_distribution = True
     package.location = "https://files.pythonhosted.org/packages/62/55/7f118b9c1b23ec15ca05d15a578d8207aa1706bc6f7c87218efffbbf875d/azure_common-1.1.28-py2.py3-none-any.whl"
-    get_source.get_source(package, 2)
+    get_source.get_source(sys.executable, package, 2)
     assert os.path.exists("dist_temp/2/azure_common-1.1.28")
     assert package.path.source_folder == "dist_temp\\2\\azure_common-1.1.28"
     assert os.path.exists("dist_temp/2/azure_common-1.1.28-py2.py3-none-any.whl") is False
@@ -127,7 +128,7 @@ def test_get_source_dist_file_tar(init_package_info):
     package.extra_index_url = None
     package.prefer_source_distribution = True
     package.location = "https://files.pythonhosted.org/packages/fa/19/43a9eb812b4d6071fdc2c55640318f7eb5a1be8dbd3b6f9d96a1996e1bb6/azure-core-1.29.4.tar.gz"
-    get_source.get_source(package, 3)
+    get_source.get_source(sys.executable, package, 3)
     assert os.path.exists("dist_temp/3/azure-core-1.29.4")
     assert package.path.source_folder == "dist_temp\\3\\azure-core-1.29.4"
     assert os.path.exists("dist_temp/3/azure-core-1.29.4.tar.gz") is False
@@ -144,7 +145,7 @@ def test_get_source_pip_whl(init_package_info):
     package.build_in_subpackage = False
     package.extra_index_url = None
     package.prefer_source_distribution = False
-    get_source.get_source(package, 4)
+    get_source.get_source(sys.executable, package, 4)
     assert os.path.exists("dist_temp/4/azure_common-1.1.28")
     assert package.path.source_folder == "dist_temp\\4\\azure_common-1.1.28"
     assert os.path.exists("dist_temp/4/azure_common-1.1.28-py2.py3-none-any.whl") is False
@@ -161,7 +162,7 @@ def test_get_source_pip_zip(init_package_info):
     package.build_in_subpackage = False
     package.extra_index_url = None
     package.prefer_source_distribution = True
-    get_source.get_source(package, 5)
+    get_source.get_source(sys.executable, package, 5)
     assert os.path.exists("dist_temp/5/azure-common-1.1.28")
     assert package.path.source_folder == "dist_temp\\5\\azure-common-1.1.28"
     assert os.path.exists("dist_temp/5/azure-common-1.1.28.zip") is False
@@ -178,7 +179,7 @@ def test_get_source_zip_file_at_position_0(init_package_info):
     package.build_in_subpackage = False
     package.extra_index_url = None
     package.prefer_source_distribution = True
-    get_source.get_source(package, 6)
+    get_source.get_source(sys.executable, package, 6)
     assert os.path.exists("dist_temp/6/azure_template-0.1.0b3942895")
     assert package.path.source_folder == "dist_temp\\6\\azure_template-0.1.0b3942895"
     assert os.path.exists("dist_temp/6/azure_template-0.1.0b3942895.tar.gz") is False

{py2docfx-0.1.10.dev1870753.dist-info → py2docfx-0.1.10.dev1871193.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: py2docfx
-Version: 0.1.10.dev1870753
+Version: 0.1.10.dev1871193
 Summary: A package built based on Sphinx which download source code package and generate yaml files supported by docfx.
 Author: Microsoft Corporation
 License: MIT License

{py2docfx-0.1.10.dev1870753.dist-info → py2docfx-0.1.10.dev1871193.dist-info}/RECORD

@@ -1,10 +1,10 @@
 py2docfx/__init__.py,sha256=kPRhPGPC1JknDotkksG428c1iIgfFr_4_7Jm-llrowY,72
-py2docfx/__main__.py,sha256=l0CbFZzaDitZHv84xL99FQ9YBNIRxapQ4MF3nu_7mEI,15137
+py2docfx/__main__.py,sha256=NeNJ-Vu1C16P9vTYhp1TcujtK52QQRPsjnnV0nBKnss,15794
 py2docfx/convert_prepare/__init__.py,sha256=XxtxrP0kmW3ZBHIAoxsPDEHzcgeC0WSnole8Lk6CjKs,11
-py2docfx/convert_prepare/environment.py,sha256=gJTGV632VAbOiYbWbLLv47wP8ffjnKLa2yFPH3wBWiQ,3583
+py2docfx/convert_prepare/environment.py,sha256=ihivOlbJ_AipP2OrgrvYcaE3EBYg4yqqsS4LzFl54m0,3624
 py2docfx/convert_prepare/generate_conf.py,sha256=wqs6iyElzJarH-20_qEL9zvZvt5xfBMsGXSXPSZy6wg,2295
 py2docfx/convert_prepare/generate_document.py,sha256=cLfFr7od0RHADkXyh2gaPreXZGvKNBUlduBosYTuPbk,3118
-py2docfx/convert_prepare/get_source.py,sha256=eEIbfLkGdBSOIJElMBTiXkKlkujb_hdFu5g6InNroTg,5851
+py2docfx/convert_prepare/get_source.py,sha256=hvWR_yPlfliLI9wywtnimKpjhJkx-LUYD6adGHgYTFA,5899
 py2docfx/convert_prepare/git.py,sha256=xGJp2nDWLfVljrxyPnFKPoLIqmBh6by-QdITogIuxi0,5893
 py2docfx/convert_prepare/install_package.py,sha256=hATmgazcSX7k2n4jQXh9sQMyNUc1k1YqHv5K5UMALq4,262
 py2docfx/convert_prepare/pack.py,sha256=vZS67_GzEhUmZWHU1dxm8gnWyRBs-kB6-KjX1d_FdOU,1260
@@ -24,7 +24,7 @@ py2docfx/convert_prepare/subpackage_merge/merge_root_package.py,sha256=uK96qL2as
 py2docfx/convert_prepare/subpackage_merge/merge_toc.py,sha256=nkVqe8R0m8D6cyTYV7aIpMDXorvn4-LXfU_vIK_hJBg,1706
 py2docfx/convert_prepare/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 py2docfx/convert_prepare/tests/test_generate_document.py,sha256=FpxwyRQFBfEdYUH16tpcJrqTAC0J5mq-Edn-N7Lz0rY,2413
-py2docfx/convert_prepare/tests/test_get_source.py,sha256=70EVlIZe71aDYDOXMnhqzZclKL8KuIg4t-X0U-pVyhA,7429
+py2docfx/convert_prepare/tests/test_get_source.py,sha256=pry20HGWLfQGDtpNoj2uc_y1AwgNGWEvBWPpJUEVSVY,7601
 py2docfx/convert_prepare/tests/test_pack.py,sha256=46JWMNzknIptDVs7D3CuxcmqBr_OKMmaw1br9H7wqco,4134
 py2docfx/convert_prepare/tests/test_package_info.py,sha256=3B-IzmUjETVO-5s3g3Lmh2E6JgopwnRauv8mB-SDZEM,3361
 py2docfx/convert_prepare/tests/test_params.py,sha256=C-aY9xDjsNTcGQATco_J_i4nF4JSmaeFPEMI3-NtiGE,2229
@@ -1882,7 +1882,7 @@ py2docfx/venv/venv0/Scripts/rst2s5.py,sha256=lnv7lRNv1Oz3gjI5BOJlTqfCJQLJxgwJ9l3
 py2docfx/venv/venv0/Scripts/rst2xetex.py,sha256=5_9VjwA5RKUMJG9L__1ZiS8DIeLHRdPWeWW7B4otM8M,909
 py2docfx/venv/venv0/Scripts/rst2xml.py,sha256=hU2nzGVbXbQBnH_ydI5f2VDkGQyDMUdTgz3rfa45JMg,638
 py2docfx/venv/venv0/Scripts/rstpep2html.py,sha256=uLmCJKfEKPeeWS0n76HharN9YEYIuCRQQXfPPBJaRso,706
-py2docfx-0.1.10.dev1870753.dist-info/METADATA,sha256=vWePDertTTMqAuxEcNX9IOfZ3sLfT68TEprsg62yYk0,601
-py2docfx-0.1.10.dev1870753.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-py2docfx-0.1.10.dev1870753.dist-info/top_level.txt,sha256=5dH2uP81dczt_qQJ38wiZ-gzoVWasfiJALWRSjdbnYU,9
-py2docfx-0.1.10.dev1870753.dist-info/RECORD,,
+py2docfx-0.1.10.dev1871193.dist-info/METADATA,sha256=z9gHaJOpywGaoBDTaHhjL33nrLivlXQilUFNMiy30KU,601
+py2docfx-0.1.10.dev1871193.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+py2docfx-0.1.10.dev1871193.dist-info/top_level.txt,sha256=5dH2uP81dczt_qQJ38wiZ-gzoVWasfiJALWRSjdbnYU,9
+py2docfx-0.1.10.dev1871193.dist-info/RECORD,,