py2docfx 0.1.10.dev1871296__py3-none-any.whl → 0.1.10.dev1871906__py3-none-any.whl

py2docfx/__main__.py

@@ -248,9 +248,9 @@ async def donwload_package_generate_documents(
     env_remove_tasks = []
 
     start_time = time.time()
-    for idx in range(min([py2docfxEnvironment.VENV_BUFFER, len(package_info_list)])):
-        env_prepare_tasks.append(asyncio.create_task(
-            py2docfxEnvironment.prepare_venv(idx, required_package_list, github_token, ado_token)))
+    await py2docfxEnvironment.prepare_venv(0, required_package_list, github_token, ado_token)
+    for idx in range(1, min([py2docfxEnvironment.VENV_BUFFER, len(package_info_list)])):
+        env_prepare_tasks.append(asyncio.create_task(py2docfxEnvironment.copy_venv(0, idx)))
     end_time = time.time()
     print(f"<warmup_venvs>{end_time-start_time}<warmup_venvs/>")
 
@@ -258,11 +258,12 @@ async def donwload_package_generate_documents(
         package_number = start_num + idx
         print(f"Processing package {package.name}, env_prepare_tasks: {len(env_prepare_tasks)}")
         start_time = time.time()
-        try:
-            await env_prepare_tasks[idx]
-        except Exception as e:
-            print(f"Failed to setup venv for package {package.name}: {e}")
-            raise
+        if idx >= 1:
+            try:
+                await env_prepare_tasks[idx-1]
+            except Exception as e:
+                print(f"Failed to setup venv for package {package.name}: {e}")
+                raise
         end_time = time.time()
         print(f"<wait_prepare_venv>{package.name},{end_time-start_time}<wait_prepare_venv/>")
 
@@ -305,16 +306,10 @@ async def donwload_package_generate_documents(
         if idx + py2docfxEnvironment.VENV_BUFFER < len(package_info_list):
             print(f"Creating venv {idx + py2docfxEnvironment.VENV_BUFFER}")
             env_prepare_tasks.append(
-                asyncio.create_task(
-                    py2docfxEnvironment.prepare_venv(
-                        idx + py2docfxEnvironment.VENV_BUFFER, 
-                        required_package_list, 
-                        github_token, 
-                        ado_token)
-                )
-            )
+                asyncio.create_task(py2docfxEnvironment.copy_venv(0, idx + py2docfxEnvironment.VENV_BUFFER)))
         end_time = time.time()
         print(f"<create_prepare_venv>{package.name},{end_time-start_time}<create_prepare_venv/>")
+        # TODO: remove the last venv
         if idx >= 1:
             start_time = time.time()
             env_remove_tasks.append(asyncio.create_task(

py2docfx/convert_prepare/environment.py

@@ -2,6 +2,7 @@ import subprocess
 import asyncio
 import os
 import shutil
+import platform
 from py2docfx import PACKAGE_ROOT
 from py2docfx.convert_prepare.package_info import PackageInfo
 from py2docfx.convert_prepare.get_source import get_source
@@ -9,7 +10,7 @@ from py2docfx.convert_prepare import pip_utils
 
 REQUIREMENT_MODULES = ["setuptools", "sphinx==6.1.3", "pyyaml", "jinja2==3.0.3", "wheel"]
 VENV_DIR = "venv"
-VENV_BUFFER = 10
+VENV_BUFFER = 5
 VENV_DELETE_BUFFER = 10
 PIP_INSTALL_COMMAND = ["-m", "pip", "install", "--upgrade"]
 
@@ -73,16 +74,37 @@ async def prepare_venv(venv_num: int, required_package_list: list[PackageInfo],
     await install_required_packages(get_venv_exe(venv_num), required_package_list, github_token, ado_token)
     print(f"<CI INFO>: venv{venv_num} setup complete.")
 
+async def copy_venv(venv_num: int, new_venv_num: int):
+    print(f"<CI INFO>: Copying venv{venv_num} to venv{new_venv_num}...")
+    venv_path = get_venv_path(venv_num)
+    new_venv_path = get_venv_path(new_venv_num)
+    if os.path.exists(venv_path):
+        if os.name == 'nt':
+            command = f'xcopy /E /I "{venv_path}" "{new_venv_path}"'
+        else:
+            command = f'cp -r "{venv_path}" "{new_venv_path}"'
+        
+        process = await asyncio.create_subprocess_shell(
+            command,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE
+        )
+        stdout, stderr = await process.communicate()
+        if process.returncode == 0:
+            print(f"<CI INFO>: venv{venv_num} copied to venv{new_venv_num}.")
+        else:
+            print(f"<CI ERROR>: Failed to copy venv{venv_num} to venv{new_venv_num}. Error: {stderr.decode()}")
+
 async def remove_environment(venv_num: int):
     venv_path = get_venv_path(venv_num)
     if os.path.exists(venv_path):
         print(f"<CI INFO>: Removing venv{venv_num}...")
         # Create a subprocess to run the shell command for removing the directory
-        process = await(await asyncio.create_subprocess_shell(
+        process = await asyncio.create_subprocess_shell(
             f'rm -rf {venv_path}' if os.name != 'nt' else f'rmdir /S /Q {venv_path}',
             stdout=asyncio.subprocess.PIPE,
             stderr=asyncio.subprocess.PIPE
-        )).wait()
+        )
         stdout, stderr = await process.communicate()
         if process.returncode == 0:
             print(f"<CI INFO>: venv{venv_num} removed.")

{py2docfx-0.1.10.dev1871296.dist-info → py2docfx-0.1.10.dev1871906.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: py2docfx
-Version: 0.1.10.dev1871296
+Version: 0.1.10.dev1871906
 Summary: A package built based on Sphinx which download source code package and generate yaml files supported by docfx.
 Author: Microsoft Corporation
 License: MIT License

{py2docfx-0.1.10.dev1871296.dist-info → py2docfx-0.1.10.dev1871906.dist-info}/RECORD

@@ -1,7 +1,7 @@
 py2docfx/__init__.py,sha256=kPRhPGPC1JknDotkksG428c1iIgfFr_4_7Jm-llrowY,72
-py2docfx/__main__.py,sha256=3vyfQ8XV8TtKPCb7C7A0d-VnzKXb58zfHOz806lUBa0,16554
+py2docfx/__main__.py,sha256=OTRwbR7oc0RRg4xrUTe5SIvG00WX1c63O5ZHDTJPf0I,16466
 py2docfx/convert_prepare/__init__.py,sha256=XxtxrP0kmW3ZBHIAoxsPDEHzcgeC0WSnole8Lk6CjKs,11
-py2docfx/convert_prepare/environment.py,sha256=PDQfOUHEHFIZUiBPDK1zsrOM_dnC0HES-IpClk7a_us,4140
+py2docfx/convert_prepare/environment.py,sha256=F_2YRI7ljMoJ4aZuT0UmceRd7qyfLWRTXq8mbhPUI34,5065
 py2docfx/convert_prepare/generate_conf.py,sha256=wqs6iyElzJarH-20_qEL9zvZvt5xfBMsGXSXPSZy6wg,2295
 py2docfx/convert_prepare/generate_document.py,sha256=cLfFr7od0RHADkXyh2gaPreXZGvKNBUlduBosYTuPbk,3118
 py2docfx/convert_prepare/get_source.py,sha256=hvWR_yPlfliLI9wywtnimKpjhJkx-LUYD6adGHgYTFA,5899
@@ -1882,7 +1882,7 @@ py2docfx/venv/venv0/Scripts/rst2s5.py,sha256=lnv7lRNv1Oz3gjI5BOJlTqfCJQLJxgwJ9l3
 py2docfx/venv/venv0/Scripts/rst2xetex.py,sha256=5_9VjwA5RKUMJG9L__1ZiS8DIeLHRdPWeWW7B4otM8M,909
 py2docfx/venv/venv0/Scripts/rst2xml.py,sha256=hU2nzGVbXbQBnH_ydI5f2VDkGQyDMUdTgz3rfa45JMg,638
 py2docfx/venv/venv0/Scripts/rstpep2html.py,sha256=uLmCJKfEKPeeWS0n76HharN9YEYIuCRQQXfPPBJaRso,706
-py2docfx-0.1.10.dev1871296.dist-info/METADATA,sha256=bblQMXf6CUjCcgJcRO1yU0S01CIXfig8J_Rpdk6-FbQ,601
-py2docfx-0.1.10.dev1871296.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-py2docfx-0.1.10.dev1871296.dist-info/top_level.txt,sha256=5dH2uP81dczt_qQJ38wiZ-gzoVWasfiJALWRSjdbnYU,9
-py2docfx-0.1.10.dev1871296.dist-info/RECORD,,
+py2docfx-0.1.10.dev1871906.dist-info/METADATA,sha256=2YF3fc3dxMD1CGCpbi-0EJibiq7hvI9JXX1VWw-pVCU,601
+py2docfx-0.1.10.dev1871906.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+py2docfx-0.1.10.dev1871906.dist-info/top_level.txt,sha256=5dH2uP81dczt_qQJ38wiZ-gzoVWasfiJALWRSjdbnYU,9
+py2docfx-0.1.10.dev1871906.dist-info/RECORD,,