py2docfx 0.1.11rc1996319__py3-none-any.whl → 0.1.12.dev2002521__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. py2docfx/__main__.py +24 -15
  2. py2docfx/convert_prepare/environment.py +13 -10
  3. py2docfx/convert_prepare/generate_document.py +6 -6
  4. py2docfx/convert_prepare/get_source.py +7 -7
  5. py2docfx/convert_prepare/git.py +10 -13
  6. py2docfx/convert_prepare/install_package.py +2 -2
  7. py2docfx/convert_prepare/pack.py +7 -10
  8. py2docfx/convert_prepare/package_info.py +3 -3
  9. py2docfx/convert_prepare/pip_utils.py +12 -14
  10. py2docfx/convert_prepare/post_process/merge_toc.py +3 -2
  11. py2docfx/convert_prepare/sphinx_caller.py +34 -12
  12. py2docfx/convert_prepare/tests/test_environment.py +0 -3
  13. py2docfx/convert_prepare/tests/test_generate_document.py +4 -2
  14. py2docfx/convert_prepare/tests/test_get_source.py +22 -14
  15. py2docfx/convert_prepare/tests/test_pack.py +6 -3
  16. py2docfx/convert_prepare/tests/test_params.py +0 -1
  17. py2docfx/convert_prepare/tests/test_sphinx_caller.py +10 -8
  18. py2docfx/convert_prepare/tests/test_subpackage.py +1 -0
  19. py2docfx/docfx_yaml/build_finished.py +1 -1
  20. py2docfx/docfx_yaml/logger.py +56 -55
  21. py2docfx/venv/venv1/Lib/site-packages/cachetools/__init__.py +7 -128
  22. py2docfx/venv/venv1/Lib/site-packages/cachetools/_decorators.py +152 -0
  23. {py2docfx-0.1.11rc1996319.dist-info → py2docfx-0.1.12.dev2002521.dist-info}/METADATA +1 -1
  24. {py2docfx-0.1.11rc1996319.dist-info → py2docfx-0.1.12.dev2002521.dist-info}/RECORD +26 -25
  25. {py2docfx-0.1.11rc1996319.dist-info → py2docfx-0.1.12.dev2002521.dist-info}/WHEEL +0 -0
  26. {py2docfx-0.1.11rc1996319.dist-info → py2docfx-0.1.12.dev2002521.dist-info}/top_level.txt +0 -0
py2docfx/__main__.py CHANGED
@@ -36,9 +36,8 @@ async def donwload_package_generate_documents(
36
36
  py2docfxEnvironment.prepare_base_venv(required_package_list, github_token, ado_token))
37
37
 
38
38
  for idx, package in enumerate(package_info_list):
39
- os.environ['PROCESSING_PACKAGE_NAME'] = package.name
40
39
  package_number = start_num + idx
41
- py2docfx_logger = py2docfxLogger.get_package_logger(__name__)
40
+ py2docfx_logger = py2docfxLogger.get_logger(__name__)
42
41
  msg = f"Processing package {package.name}, env_prepare_tasks: {len(env_prepare_tasks)}"
43
42
  py2docfx_logger.info(msg)
44
43
 
@@ -47,11 +46,11 @@ async def donwload_package_generate_documents(
47
46
  except Exception as e:
48
47
  msg = f"Failed to setup venv for package {package.name}: {e}"
49
48
  py2docfx_logger.error(msg)
50
- raise
49
+ raise e
51
50
 
52
- generate_document(package, output_root,
51
+ await generate_document(package, output_root,
53
52
  py2docfxEnvironment.get_base_venv_sphinx_build_path(),
54
- py2docfxEnvironment.get_venv_package_path(idx),
53
+ py2docfxEnvironment.get_venv_package_path(idx),
55
54
  py2docfxEnvironment.get_base_venv_exe())
56
55
 
57
56
  merge_toc(YAML_OUTPUT_ROOT, package.path.yaml_output_folder)
@@ -80,7 +79,7 @@ async def donwload_package_generate_documents(
80
79
  msg = f"Removing venv {idx-py2docfxEnvironment.VENV_BUFFER}"
81
80
  py2docfx_logger.info(msg)
82
81
  await env_remove_tasks[idx-py2docfxEnvironment.VENV_BUFFER]
83
-
82
+
84
83
  if output_doc_folder:
85
84
  move_root_toc_to_target(YAML_OUTPUT_ROOT, output_doc_folder)
86
85
 
@@ -88,17 +87,28 @@ async def donwload_package_generate_documents(
88
87
  if env_remove_tasks[idx] != None and not env_remove_tasks[idx].done():
89
88
  await env_remove_tasks[idx]
90
89
 
91
- def main(argv) -> int:
90
+ def fishish_up():
91
+ warning_count, error_count = py2docfxLogger.get_warning_error_count()
92
+ py2docfxLogger.output_log_by_log_level()
93
+ print(f"Warning count: {warning_count}, Error count: {error_count}")
94
+ logging.shutdown()
95
+
96
+ async def main(argv) -> int:
92
97
  # TODO: may need to purge pip cache
93
98
  (package_info_list,
94
99
  required_package_list,
95
100
  github_token, ado_token,
96
101
  output_root, verbose,
97
102
  show_warning) = parse_command_line_args(argv)
98
-
103
+
99
104
  clean_up_folder_list = [py2docfxEnvironment.VENV_DIR, DIST_TEMP, SOURCE_REPO, TARGET_REPO, LOG_FOLDER]
100
105
  temp_folder_clean_up(clean_up_folder_list)
101
106
 
107
+ # create log folder and package log folder
108
+ log_folder = os.path.join(PACKAGE_ROOT, LOG_FOLDER)
109
+ os.makedirs(log_folder)
110
+ os.makedirs(os.path.join(log_folder, "package_logs"))
111
+
102
112
  py2docfxLogger.decide_global_log_level(verbose, show_warning)
103
113
 
104
114
  py2docfx_logger = py2docfxLogger.get_logger(__name__)
@@ -110,19 +120,18 @@ def main(argv) -> int:
110
120
  output_doc_folder = prepare_out_dir(output_root)
111
121
 
112
122
  try:
113
- asyncio.run(donwload_package_generate_documents(
123
+ await donwload_package_generate_documents(
114
124
  package_info_list, output_root, output_doc_folder,
115
- github_token, ado_token, required_package_list))
125
+ github_token, ado_token, required_package_list)
116
126
  except Exception as e:
117
127
  msg = f"An error occurred: {e}"
118
128
  py2docfx_logger.error(msg)
129
+ fishish_up()
130
+ asyncio.get_event_loop().stop()
119
131
  raise
120
132
 
121
- warning_count, error_count = py2docfxLogger.get_warning_error_count()
122
- py2docfxLogger.output_log_by_log_level()
123
- print(f"Warning count: {warning_count}, Error count: {error_count}")
124
- logging.shutdown()
133
+ fishish_up()
125
134
  return 0
126
135
 
127
136
  if __name__ == "__main__":
128
- sys.exit(main(sys.argv[1:]))
137
+ sys.exit(asyncio.run(main(sys.argv[1:])))
@@ -22,7 +22,7 @@ PIP_INSTALL_VENV_COMMON_OPTIONS = [
22
22
  "--disable-pip-version-check",
23
23
  ]
24
24
 
25
- def install_converter_requirements(executable: str):
25
+ async def install_converter_requirements(executable: str):
26
26
  """
27
27
  Install setuptools/sphinx/pyyaml/jinja2
28
28
  Replacing logic of
@@ -41,14 +41,14 @@ def install_converter_requirements(executable: str):
41
41
  for module in REQUIREMENT_MODULES:
42
42
  msg = f"<CI INFO>: Upgrading {module}..."
43
43
  py2docfx_logger.info(msg)
44
- subprocess.run(
45
- pip_install_cmd + [module] + pip_install_common_options, check=True
46
- )
44
+ full_cmd = pip_install_cmd + [module] + pip_install_common_options
45
+ await run_async_subprocess(executable, full_cmd, py2docfx_logger)
47
46
 
48
47
  async def install_venv_requirements(venv_num: int):
49
48
  venv_exe = get_venv_exe(venv_num)
50
49
  pip_cmd = ["-m", "pip", "install", "--upgrade"]+ VENV_REQUIREMENT_MODULES
51
- await run_async_subprocess(venv_exe, pip_cmd)
50
+ py2docfx_logger = get_logger(__name__)
51
+ await run_async_subprocess(venv_exe, pip_cmd, py2docfx_logger)
52
52
 
53
53
  def get_venv_path(venv_num: int) -> str:
54
54
  return os.path.join(PACKAGE_ROOT, VENV_DIR, "venv"+str(venv_num))
@@ -85,7 +85,8 @@ def get_base_venv_sphinx_build_path() -> str:
85
85
 
86
86
  async def install_converter_requirement_async(executable: str):
87
87
  pip_cmd = PIP_INSTALL_COMMAND + PIP_INSTALL_VENV_COMMON_OPTIONS + REQUIREMENT_MODULES
88
- await run_async_subprocess(executable, pip_cmd)
88
+ py2docfx_logger = get_logger(__name__)
89
+ await run_async_subprocess(executable, pip_cmd, py2docfx_logger)
89
90
 
90
91
  async def install_required_packages(
91
92
  executable: str, required_package_list: list[PackageInfo], github_token: str, ado_token: str):
@@ -95,7 +96,8 @@ async def install_required_packages(
95
96
  # get_source(package, idx, vststoken=ado_token, githubtoken=github_token)
96
97
  package_name, options = package.get_install_command()
97
98
  pip_cmd = PIP_INSTALL_COMMAND + PIP_INSTALL_VENV_COMMON_OPTIONS + options + [package_name]
98
- await run_async_subprocess(executable, pip_cmd)
99
+ py2docfx_logger = get_logger(__name__)
100
+ await run_async_subprocess(executable, pip_cmd, py2docfx_logger)
99
101
 
100
102
  async def create_environment(venv_path: int):
101
103
  if os.name == 'nt':
@@ -125,7 +127,7 @@ async def prepare_venv(venv_num: int, package_info: PackageInfo, package_number:
125
127
  py2docfx_logger = get_logger(__name__)
126
128
  await create_environment(get_venv_path(venv_num))
127
129
  await install_venv_requirements(venv_num)
128
- get_source(get_venv_exe(venv_num), package_info, package_number, vststoken=ado_token, githubtoken=github_token)
130
+ await get_source(get_venv_exe(venv_num), package_info, package_number, vststoken=ado_token, githubtoken=github_token)
129
131
  package_name, options = package_info.get_install_command()
130
132
  await pip_utils.install_in_exe_async(get_venv_exe(venv_num), package_name, options)
131
133
  msg = f"<CI INFO>: venv{venv_num} setup complete."
@@ -138,8 +140,9 @@ async def remove_environment(venv_num: int):
138
140
  msg = f"<CI INFO>: Removing venv{venv_num}..."
139
141
  py2docfx_logger.info(msg)
140
142
  # Create a subprocess to run the shell command for removing the directory
143
+ cmd = f'rm -rf {venv_path}' if os.name != 'nt' else f'rmdir /S /Q {venv_path}'
141
144
  process = await asyncio.create_subprocess_shell(
142
- f'rm -rf {venv_path}' if os.name != 'nt' else f'rmdir /S /Q {venv_path}',
145
+ cmd,
143
146
  stdout=asyncio.subprocess.PIPE,
144
147
  stderr=asyncio.subprocess.PIPE
145
148
  )
@@ -150,4 +153,4 @@ async def remove_environment(venv_num: int):
150
153
  else:
151
154
  msg = f"<CI ERROR>: Failed to remove venv{venv_num}. Error: {stderr.decode()}"
152
155
  py2docfx_logger.error(msg)
153
- raise RuntimeError()
156
+ raise subprocess.CalledProcessError(process.returncode, cmd, stdout, stderr)
@@ -10,8 +10,8 @@ from py2docfx.convert_prepare.subpackage import merge_subpackage_files
10
10
 
11
11
  CONF_TEMPLATE_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "conf_templates")
12
12
 
13
- def generate_document(pkg: PackageInfo, output_root: str | os.PathLike, sphinx_build_path: str, extra_package_path: str, executable=sys.executable):
14
- py2docfx_logger = get_package_logger(__name__)
13
+ async def generate_document(pkg: PackageInfo, output_root: str | os.PathLike, sphinx_build_path: str, extra_package_path: str, executable=sys.executable):
14
+ py2docfx_logger = get_package_logger(__name__, pkg.name)
15
15
  # Copy manual written RST from target doc repo
16
16
  package_paths = pkg.path
17
17
  if output_root:
@@ -28,12 +28,12 @@ def generate_document(pkg: PackageInfo, output_root: str | os.PathLike, sphinx_b
28
28
  exclude_paths = pkg.get_exluded_command()
29
29
 
30
30
  if pkg.install_type == pkg.InstallType.SOURCE_CODE and getattr(pkg, "branch", None):
31
- checkout(package_paths.source_folder, pkg.branch)
31
+ await checkout(package_paths.source_folder, pkg.branch)
32
32
 
33
33
  msg = f"<CI INFO>: Generating RST files for {pkg.name}."
34
34
  py2docfx_logger.info(msg)
35
35
 
36
- subpackages_rst_record = run_apidoc(package_paths.doc_folder, package_paths.source_folder,
36
+ subpackages_rst_record = await run_apidoc(pkg.name, package_paths.doc_folder, package_paths.source_folder,
37
37
  exclude_paths, pkg)
38
38
 
39
39
  msg = f"<CI INFO>: Listing RST files:"
@@ -45,7 +45,7 @@ def generate_document(pkg: PackageInfo, output_root: str | os.PathLike, sphinx_b
45
45
  py2docfx_logger.info(msg)
46
46
 
47
47
  generate_conf(pkg, package_paths.doc_folder, CONF_TEMPLATE_DIR)
48
- run_converter(package_paths.doc_folder, package_paths.yaml_output_folder, sphinx_build_path, extra_package_path, executable=executable)
48
+ await run_converter(pkg.name, package_paths.doc_folder, package_paths.yaml_output_folder, sphinx_build_path, extra_package_path, executable=executable)
49
49
 
50
50
  subpackages_path_record = {}
51
51
  if pkg.build_in_subpackage:
@@ -53,6 +53,6 @@ def generate_document(pkg: PackageInfo, output_root: str | os.PathLike, sphinx_b
53
53
  for (subpackage_name, subpackage_path) in subpackages_rst_record.items():
54
54
  subpackage_yaml_path = os.path.join(subpackages_yaml_path, subpackage_name)
55
55
  subpackages_path_record[subpackage_name] = subpackage_yaml_path
56
- run_converter(subpackage_path, subpackage_yaml_path, sphinx_build_path, extra_package_path, executable=executable)
56
+ await run_converter(pkg.name, subpackage_path, subpackage_yaml_path, sphinx_build_path, extra_package_path, executable=executable)
57
57
 
58
58
  merge_subpackage_files(subpackages_path_record, package_paths.yaml_output_folder, pkg.name)
@@ -72,7 +72,7 @@ def update_package_info(executable: str, pkg: PackageInfo, source_folder: str):
72
72
  source_folder=source_folder, yaml_output_folder=yaml_output_folder, package_name=pkg.name
73
73
  )
74
74
 
75
- def get_source(executable: str, pkg: PackageInfo, cnt: int, vststoken=None, githubtoken=None):
75
+ async def get_source(executable: str, pkg: PackageInfo, cnt: int, vststoken=None, githubtoken=None):
76
76
  py2docfx_logger = get_logger(__name__)
77
77
  path_cnt = str(cnt)
78
78
  dist_dir = path.join(DIST_TEMP, path_cnt)
@@ -81,7 +81,7 @@ def get_source(executable: str, pkg: PackageInfo, cnt: int, vststoken=None, gith
81
81
  if pkg.url:
82
82
  repo_folder = path.join(SOURCE_REPO, path_cnt)
83
83
  token = githubtoken if "github.com" in pkg.url else vststoken
84
- source_folder = git.clone(
84
+ source_folder = await git.clone(
85
85
  repo_location=pkg.url,
86
86
  branch=pkg.branch,
87
87
  folder=repo_folder,
@@ -94,7 +94,7 @@ def get_source(executable: str, pkg: PackageInfo, cnt: int, vststoken=None, gith
94
94
  sys.path.insert(0, source_folder)
95
95
  elif pkg.install_type == PackageInfo.InstallType.PYPI:
96
96
  full_name = pkg.get_combined_name_version()
97
- pip_utils.download(
97
+ await pip_utils.download(
98
98
  full_name,
99
99
  dist_dir,
100
100
  extra_index_url=pkg.extra_index_url,
@@ -102,17 +102,17 @@ def get_source(executable: str, pkg: PackageInfo, cnt: int, vststoken=None, gith
102
102
  )
103
103
  # unpack the downloaded wheel file.
104
104
  downloaded_dist_file = path.join(dist_dir, os.listdir(dist_dir)[0])
105
- pack.unpack_dist(downloaded_dist_file)
105
+ await pack.unpack_dist(pkg.name, downloaded_dist_file)
106
106
  os.remove(downloaded_dist_file)
107
107
  source_folder = path.join(
108
108
  path.dirname(downloaded_dist_file),
109
109
  os.listdir(dist_dir)[0]
110
110
  )
111
111
  elif pkg.install_type == PackageInfo.InstallType.DIST_FILE:
112
- pip_utils.download(pkg.location, dist_dir, prefer_source_distribution=False)
112
+ await pip_utils.download(pkg.location, dist_dir, prefer_source_distribution=False)
113
113
  # unpack the downloaded dist file.
114
114
  downloaded_dist_file = path.join(dist_dir, os.listdir(dist_dir)[0])
115
- pack.unpack_dist(downloaded_dist_file)
115
+ await pack.unpack_dist(pkg.name, downloaded_dist_file)
116
116
  os.remove(downloaded_dist_file)
117
117
  if downloaded_dist_file.endswith(".tar.gz"):
118
118
  downloaded_dist_file = downloaded_dist_file.rsplit(".", maxsplit=1)[
@@ -124,6 +124,6 @@ def get_source(executable: str, pkg: PackageInfo, cnt: int, vststoken=None, gith
124
124
  else:
125
125
  msg = f"Unknown install type: {pkg.install_type}"
126
126
  py2docfx_logger.error(msg)
127
- raise ValueError()
127
+ raise ValueError(msg)
128
128
 
129
129
  update_package_info(executable, pkg, source_folder)
@@ -1,11 +1,11 @@
1
1
  import re
2
2
  import subprocess
3
3
 
4
- from py2docfx.docfx_yaml.logger import get_logger, log_git_clone_subprocess_ouput
4
+ from py2docfx.docfx_yaml.logger import get_logger, run_async_subprocess_without_executable
5
5
 
6
6
  repoMap = {}
7
7
 
8
- def clone(repo_location, branch, folder, extra_token=None):
8
+ async def clone(repo_location, branch, folder, extra_token=None):
9
9
  """
10
10
  Clone a git repo to the folder
11
11
  When extra_token isn't None, add it to the command
@@ -22,7 +22,7 @@ def clone(repo_location, branch, folder, extra_token=None):
22
22
  if not test_url(repo_location, extra_token):
23
23
  msg = "Git repo address {} is not a valid URL.".format(repo_location)
24
24
  py2docfx_logger.error(msg)
25
- raise ValueError()
25
+ raise ValueError(msg)
26
26
  else:
27
27
  # Remove http(s):// from url to record. Further avoid dup-clone.
28
28
  pureURL = re.sub("^\s*https?://", "", repo_location)
@@ -57,8 +57,7 @@ def clone(repo_location, branch, folder, extra_token=None):
57
57
  extra_token
58
58
  ),
59
59
  ]
60
- output = subprocess.run(clone_params, check=True, capture_output=True, text=True)
61
- log_git_clone_subprocess_ouput(output, py2docfx_logger)
60
+ await run_async_subprocess_without_executable(clone_params, py2docfx_logger)
62
61
  repoMap[pureURL] = folder
63
62
  msg = "<CI INFO>: Repo {} successfully cloned in {}...".format(
64
63
  repo_location, repoMap[pureURL]
@@ -123,7 +122,7 @@ def convertBranch(repo_url, branch, extraHeader):
123
122
  return result
124
123
 
125
124
 
126
- def checkout(folder, branch):
125
+ async def checkout(folder, branch):
127
126
  """
128
127
  Checkout one folder to a given branch
129
128
 
@@ -145,8 +144,7 @@ def checkout(folder, branch):
145
144
  if "* {}".format(branch) not in branches:
146
145
  # branch is not exactly current branch
147
146
  if branch not in branches:
148
- output = subprocess.run(
149
- [
147
+ fetch_cmd = [
150
148
  "git",
151
149
  "-C",
152
150
  folder,
@@ -154,12 +152,11 @@ def checkout(folder, branch):
154
152
  "--quiet",
155
153
  remote,
156
154
  "{}:{}".format(branch, branch),
157
- ], capture_output=True, text=True
158
- )
159
- log_subprocess_ouput(output, py2docfx_logger)
155
+ ]
156
+ await run_async_subprocess_without_executable(fetch_cmd, py2docfx_logger)
160
157
 
161
- output = subprocess.run(["git", "-C", folder, "checkout", "--quiet", branch], check=True, capture_output=True, text=True)
162
- log_subprocess_ouput(output, py2docfx_logger)
158
+ checkout_cmd = ["git", "-C", folder, "checkout", "--quiet", branch]
159
+ await run_async_subprocess_without_executable(checkout_cmd, py2docfx_logger)
163
160
  msg = "<CI INFO>: Switched to branch {}.".format(branch)
164
161
  py2docfx_logger.info(msg)
165
162
 
@@ -1,6 +1,6 @@
1
1
  from py2docfx.convert_prepare.package_info import PackageInfo
2
2
  import py2docfx.convert_prepare.pip_utils as pip_utils
3
3
 
4
- def install_package(pkg: PackageInfo):
4
+ async def install_package(pkg: PackageInfo):
5
5
  package_name, options = pkg.get_install_command()
6
- pip_utils.install(package_name, options)
6
+ await pip_utils.install(package_name, options)
@@ -1,14 +1,13 @@
1
1
  import shutil
2
2
  from os import path
3
3
  import os
4
- import subprocess
5
4
  import sys
6
5
 
7
- from py2docfx.docfx_yaml.logger import get_package_logger, log_subprocess_ouput
6
+ from py2docfx.docfx_yaml.logger import get_package_logger, run_async_subprocess
8
7
 
9
- def unpack_dist(dist_file):
8
+ async def unpack_dist(package_name, dist_file):
10
9
  if dist_file.endswith(".whl"):
11
- unpack_wheel(dist_file)
10
+ await unpack_wheel(package_name, dist_file)
12
11
  else:
13
12
  unpack_compressed(dist_file)
14
13
 
@@ -20,20 +19,18 @@ def unpack_compressed(file_path):
20
19
  shutil.unpack_archive(file_path, path.dirname(file_path))
21
20
 
22
21
 
23
- def unpack_wheel(file_path):
22
+ async def unpack_wheel(package_name, file_path):
24
23
  """
25
24
  Transform a wheel file to a folder containing source code
26
25
  """
27
- py2docfx_logger = get_package_logger(__name__)
28
- command = [sys.executable,
29
- '-m',
26
+ py2docfx_logger = get_package_logger(__name__, package_name)
27
+ command = ['-m',
30
28
  'wheel',
31
29
  'unpack',
32
30
  file_path,
33
31
  '-d',
34
32
  os.path.dirname(file_path)]
35
- output = subprocess.run(command, check=True, capture_output=True, text=True)
36
- log_subprocess_ouput(output, py2docfx_logger)
33
+ await run_async_subprocess(sys.executable, command, py2docfx_logger)
37
34
  extract_folder = [file for file in os.listdir(path.dirname(file_path)) if path.isdir(path.join(path.dirname(file_path), file))][0]
38
35
  data_folder = path.join(path.dirname(file_path), extract_folder, extract_folder + ".data")
39
36
  if path.exists(data_folder):
@@ -26,7 +26,7 @@ class PackageInfo:
26
26
  name, value
27
27
  )
28
28
  py2docfx_logger.error(message)
29
- raise ValueError()
29
+ raise ValueError(message)
30
30
 
31
31
  @classmethod
32
32
  def parse_from(cls, dict, reading_required_packages=False):
@@ -64,7 +64,7 @@ class PackageInfo:
64
64
  if not package_info.folder:
65
65
  msg = "When install_type is source_code, url or folder should be provided"
66
66
  py2docfx_logger.error(msg)
67
- raise ValueError()
67
+ raise ValueError(msg)
68
68
  else:
69
69
  msg = f'Read source code from local folder: {package_info.folder}'
70
70
  py2docfx_logger.info(msg)
@@ -149,7 +149,7 @@ class PackageInfo:
149
149
  else:
150
150
  msg = "Should set source code location before build documents"
151
151
  py2docfx_logger.error(msg)
152
- raise ValueError()
152
+ raise ValueError(msg)
153
153
  exclude_path = []
154
154
  if code_location:
155
155
  exclude_path.append(os.path.join(code_location, "build/*"))
@@ -1,7 +1,7 @@
1
- import subprocess
1
+ import sys
2
2
 
3
3
  from py2docfx import PACKAGE_ROOT
4
- from py2docfx.docfx_yaml.logger import get_logger, log_subprocess_ouput, run_async_subprocess
4
+ from py2docfx.docfx_yaml.logger import get_logger, run_async_subprocess, run_async_subprocess_without_executable
5
5
 
6
6
  PYPI = "pypi"
7
7
 
@@ -14,7 +14,7 @@ pip_install_common_options = [
14
14
  "--verbose"
15
15
  ]
16
16
 
17
- def download(package_name, path, extra_index_url=None, prefer_source_distribution=True):
17
+ async def download(package_name, path, extra_index_url=None, prefer_source_distribution=True):
18
18
  # Downloads a package from PyPI to the specified path using pip.
19
19
  download_param = ["pip", "download", "--dest", path, "--no-deps", package_name]
20
20
  if extra_index_url:
@@ -24,29 +24,27 @@ def download(package_name, path, extra_index_url=None, prefer_source_distributio
24
24
  download_param.append("--no-binary=:all:")
25
25
  else:
26
26
  download_param.append("--prefer-binary")
27
- output = subprocess.run(download_param, check=True, cwd=PACKAGE_ROOT, capture_output=True, text=True)
28
- py2docfx_logger = get_logger(__name__)
29
- log_subprocess_ouput(output, py2docfx_logger)
30
27
 
28
+ py2docfx_logger = get_logger(__name__)
29
+ await run_async_subprocess_without_executable(download_param, py2docfx_logger, cwd=PACKAGE_ROOT)
31
30
 
32
- def install(package_name, options):
31
+ async def install(package_name, options):
33
32
  # Installs a package from PyPI using pip.
34
33
  install_param = "pip install {} {}".format(
35
34
  " ".join(pip_install_common_options + options), package_name
36
35
  ).split(" ")
37
- output = subprocess.run(install_param, check=True, cwd=PACKAGE_ROOT, capture_output=True, text=True)
38
36
  py2docfx_logger = get_logger(__name__)
39
- log_subprocess_ouput(output, py2docfx_logger)
37
+ await run_async_subprocess_without_executable(install_param, py2docfx_logger, cwd=PACKAGE_ROOT)
40
38
 
41
- def install_in_exe(exe_path, package_name, options):
39
+ async def install_in_exe(exe_path, package_name, options):
42
40
  # Installs a package from PyPI using pip.
43
41
  install_param = [exe_path] + "-m pip install {} {}".format(
44
42
  " ".join(pip_install_common_options + options), package_name
45
43
  ).split(" ")
46
- output = subprocess.run(install_param, check=True, cwd=PACKAGE_ROOT, capture_output=True, text=True)
47
44
  py2docfx_logger = get_logger(__name__)
48
- log_subprocess_ouput(output, py2docfx_logger)
49
-
45
+ await run_async_subprocess(exe_path, install_param, py2docfx_logger, cwd=PACKAGE_ROOT)
46
+
50
47
  async def install_in_exe_async(exe_path, package_name, options):
51
48
  pip_cmd = ["-m", "pip", "install"]+ pip_install_common_options + options + [package_name]
52
- await run_async_subprocess(exe_path, pip_cmd)
49
+ py2docfx_logger = get_logger(__name__)
50
+ await run_async_subprocess(exe_path, pip_cmd, py2docfx_logger)
@@ -21,8 +21,9 @@ def merge_toc(
21
21
  if not toc_content.endswith("\n"):
22
22
  root_toc_handle.write("\n")
23
23
  else:
24
- py2docfx_logger.error(f"TOC content empty: {package_toc_path}")
25
- raise ValueError()
24
+ msg = f"TOC content empty: {package_toc_path}"
25
+ py2docfx_logger.error(msg)
26
+ raise ValueError(msg)
26
27
 
27
28
  # delete package toc.yml
28
29
  os.remove(package_toc_path)
@@ -1,9 +1,10 @@
1
+ import asyncio
1
2
  import os
2
- import sys
3
3
  import subprocess
4
+ import sys
4
5
 
5
6
  from py2docfx import PACKAGE_ROOT
6
- from py2docfx.docfx_yaml.logger import get_package_logger,log_subprocess_ouput
7
+ from py2docfx.docfx_yaml.logger import get_package_logger,run_async_subprocess
7
8
  from py2docfx.convert_prepare.package_info import PackageInfo
8
9
  from py2docfx.convert_prepare.paths import folder_is_hidden
9
10
  from py2docfx.convert_prepare.subpackage import (get_subpackages,
@@ -11,14 +12,14 @@ from py2docfx.convert_prepare.subpackage import (get_subpackages,
11
12
 
12
13
  DEBUG_SPHINX_FLAG = 'PY2DOCFX_DEBUG_SPHINX'
13
14
 
14
- def run_apidoc(rst_path, source_code_path, exclude_paths, package_info: PackageInfo):
15
+ async def run_apidoc(package_name, rst_path, source_code_path, exclude_paths, package_info: PackageInfo):
15
16
  """
16
17
  Run sphinx-apidoc to generate RST inside rst_path folder
17
18
 
18
19
  Replacing
19
20
  https://apidrop.visualstudio.com/Content%20CI/_git/ReferenceAutomation?path=/Python/build.ps1&line=110&lineEnd=126&lineStartColumn=1&lineEndColumn=14&lineStyle=plain&_a=contents
20
21
  """
21
- py2docfx_logger = get_package_logger(__name__)
22
+ py2docfx_logger = get_package_logger(__name__, package_name)
22
23
  subfolderList = [name for name in
23
24
  os.listdir(source_code_path)
24
25
  if os.path.isdir(os.path.join(source_code_path, name))
@@ -40,8 +41,8 @@ def run_apidoc(rst_path, source_code_path, exclude_paths, package_info: PackageI
40
41
  subfolderPath,
41
42
  ]
42
43
  args.extend(exclude_paths)
43
- output = subprocess.run([sys.executable, "-m", "sphinx.ext.apidoc"] + args, capture_output=True, text=True)
44
- log_subprocess_ouput(output, py2docfx_logger)
44
+ full_args = ["-m", "sphinx.ext.apidoc"] + args
45
+ await run_async_subprocess(sys.executable, full_args, py2docfx_logger)
45
46
  if package_info.build_in_subpackage and subfolder == "azure":
46
47
  subpackages_rst_record = move_rst_files_to_subfolder(
47
48
  package_paths.doc_folder, package_info.name,
@@ -49,7 +50,13 @@ def run_apidoc(rst_path, source_code_path, exclude_paths, package_info: PackageI
49
50
  return subpackages_rst_record
50
51
 
51
52
 
52
- def run_converter(rst_path, out_path, sphinx_build_path: str, extra_package_path: str, conf_path = None, executable = sys.executable):
53
+ async def run_converter(package_name: str,
54
+ rst_path,
55
+ out_path,
56
+ sphinx_build_path: str,
57
+ extra_package_path: str,
58
+ conf_path = None,
59
+ executable = sys.executable):
53
60
  """
54
61
  Take rst files as input and run sphinx converter
55
62
 
@@ -58,7 +65,7 @@ def run_converter(rst_path, out_path, sphinx_build_path: str, extra_package_path
58
65
  Replacing
59
66
  https://apidrop.visualstudio.com/Content%20CI/_git/ReferenceAutomation?path=/Python/build.ps1&line=150&lineEnd=161&lineStartColumn=13&lineEndColumn=52&lineStyle=plain&_a=contents
60
67
  """
61
- py2docfx_logger = get_package_logger(__name__)
68
+ py2docfx_logger = get_package_logger(__name__, package_name)
62
69
  outdir = os.path.join(out_path, "_build")
63
70
 
64
71
  # Sphinx/docutils have memory leak including linecaches, module-import-caches,
@@ -67,9 +74,8 @@ def run_converter(rst_path, out_path, sphinx_build_path: str, extra_package_path
67
74
  if not sys.executable:
68
75
  msg = "Can't get the executable binary for the Python interpreter."
69
76
  py2docfx_logger.error(msg)
70
- raise ValueError()
77
+ raise ValueError(msg)
71
78
  sphinx_param = [
72
- executable,
73
79
  sphinx_build_path,
74
80
  rst_path,
75
81
  outdir,
@@ -86,6 +92,22 @@ def run_converter(rst_path, out_path, sphinx_build_path: str, extra_package_path
86
92
  env_tmp["PYTHONPATH"] = f"{extra_package_path};{package_root_parent};"
87
93
  else:
88
94
  env_tmp["PYTHONPATH"] = f"{extra_package_path}:{package_root_parent}:"
89
- output = subprocess.run(sphinx_param, check=True, cwd=PACKAGE_ROOT, env=env_tmp, capture_output=True ,text=True)
90
- log_subprocess_ouput(output, py2docfx_logger)
95
+ os.environ['PROCESSING_PACKAGE_NAME'] = package_name
96
+ proc = await asyncio.create_subprocess_exec(
97
+ executable, *sphinx_param,
98
+ cwd=PACKAGE_ROOT,
99
+ env=env_tmp,
100
+ stdout=asyncio.subprocess.PIPE,
101
+ stderr=asyncio.subprocess.PIPE
102
+ )
103
+ stdout, stderr = await proc.communicate()
104
+ return_code = proc.returncode
105
+
106
+ if return_code == 0:
107
+ py2docfx_logger.info(f"{stdout}")
108
+ py2docfx_logger.info(f"{stderr}")
109
+ else:
110
+ py2docfx_logger.error(f"{stderr}")
111
+ raise subprocess.CalledProcessError(return_code, sphinx_param, stdout, stderr)
112
+
91
113
  return outdir
@@ -1,8 +1,6 @@
1
- import asyncio
2
1
  import os
3
2
  import shutil
4
3
  import pytest
5
- import sys
6
4
 
7
5
  from py2docfx.convert_prepare.environment import create_environment
8
6
  from py2docfx.convert_prepare.environment import remove_environment
@@ -14,7 +12,6 @@ from py2docfx.convert_prepare.environment import prepare_base_venv
14
12
  from py2docfx.convert_prepare.environment import get_base_venv_exe
15
13
  from py2docfx.convert_prepare.environment import get_base_venv_path
16
14
  from py2docfx.convert_prepare.package_info import PackageInfo
17
- pytest_plugins = ('pytest_asyncio',)
18
15
 
19
16
  @pytest.mark.asyncio
20
17
  async def test_venv_creation_package_install_remove():
@@ -4,6 +4,7 @@ Test the generate_document function.
4
4
  import os
5
5
  import sys
6
6
  import shutil
7
+ import pytest
7
8
  import sphinx
8
9
  import sphinx.cmd.build
9
10
 
@@ -11,7 +12,8 @@ from py2docfx.convert_prepare.generate_document import generate_document
11
12
  from py2docfx.convert_prepare.package_info import PackageInfo
12
13
  from py2docfx.convert_prepare.source import Source
13
14
 
14
- def test_generate_document(tmp_path):
15
+ @pytest.mark.asyncio
16
+ async def test_generate_document(tmp_path):
15
17
  """
16
18
  Test the generate_document function.
17
19
  """
@@ -42,7 +44,7 @@ def test_generate_document(tmp_path):
42
44
 
43
45
  # call the function
44
46
 
45
- generate_document(package, output_root, sphinx_build_path = sphinx.cmd.build.__file__, extra_package_path = source_folder)
47
+ await generate_document(package, output_root, sphinx_build_path = sphinx.cmd.build.__file__, extra_package_path = source_folder)
46
48
 
47
49
  #assert the result
48
50
  yaml_path = os.path.join(yaml_output_folder, "_build", "docfx_yaml")