py2docfx 0.1.11rc1996319__py3-none-any.whl → 0.1.11rc1997820__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
py2docfx/__main__.py CHANGED
@@ -36,9 +36,8 @@ async def donwload_package_generate_documents(
36
36
  py2docfxEnvironment.prepare_base_venv(required_package_list, github_token, ado_token))
37
37
 
38
38
  for idx, package in enumerate(package_info_list):
39
- os.environ['PROCESSING_PACKAGE_NAME'] = package.name
40
39
  package_number = start_num + idx
41
- py2docfx_logger = py2docfxLogger.get_package_logger(__name__)
40
+ py2docfx_logger = py2docfxLogger.get_logger(__name__)
42
41
  msg = f"Processing package {package.name}, env_prepare_tasks: {len(env_prepare_tasks)}"
43
42
  py2docfx_logger.info(msg)
44
43
 
@@ -47,11 +46,11 @@ async def donwload_package_generate_documents(
47
46
  except Exception as e:
48
47
  msg = f"Failed to setup venv for package {package.name}: {e}"
49
48
  py2docfx_logger.error(msg)
50
- raise
49
+ raise e
51
50
 
52
- generate_document(package, output_root,
51
+ await generate_document(package, output_root,
53
52
  py2docfxEnvironment.get_base_venv_sphinx_build_path(),
54
- py2docfxEnvironment.get_venv_package_path(idx),
53
+ py2docfxEnvironment.get_venv_package_path(idx),
55
54
  py2docfxEnvironment.get_base_venv_exe())
56
55
 
57
56
  merge_toc(YAML_OUTPUT_ROOT, package.path.yaml_output_folder)
@@ -80,7 +79,7 @@ async def donwload_package_generate_documents(
80
79
  msg = f"Removing venv {idx-py2docfxEnvironment.VENV_BUFFER}"
81
80
  py2docfx_logger.info(msg)
82
81
  await env_remove_tasks[idx-py2docfxEnvironment.VENV_BUFFER]
83
-
82
+
84
83
  if output_doc_folder:
85
84
  move_root_toc_to_target(YAML_OUTPUT_ROOT, output_doc_folder)
86
85
 
@@ -88,17 +87,28 @@ async def donwload_package_generate_documents(
88
87
  if env_remove_tasks[idx] != None and not env_remove_tasks[idx].done():
89
88
  await env_remove_tasks[idx]
90
89
 
91
- def main(argv) -> int:
90
+ def fishish_up():
91
+ warning_count, error_count = py2docfxLogger.get_warning_error_count()
92
+ py2docfxLogger.output_log_by_log_level()
93
+ print(f"Warning count: {warning_count}, Error count: {error_count}")
94
+ logging.shutdown()
95
+
96
+ async def main(argv) -> int:
92
97
  # TODO: may need to purge pip cache
93
98
  (package_info_list,
94
99
  required_package_list,
95
100
  github_token, ado_token,
96
101
  output_root, verbose,
97
102
  show_warning) = parse_command_line_args(argv)
98
-
103
+
99
104
  clean_up_folder_list = [py2docfxEnvironment.VENV_DIR, DIST_TEMP, SOURCE_REPO, TARGET_REPO, LOG_FOLDER]
100
105
  temp_folder_clean_up(clean_up_folder_list)
101
106
 
107
+ # create log folder and package log folder
108
+ log_folder = os.path.join(PACKAGE_ROOT, LOG_FOLDER)
109
+ os.makedirs(log_folder)
110
+ os.makedirs(os.path.join(log_folder, "package_logs"))
111
+
102
112
  py2docfxLogger.decide_global_log_level(verbose, show_warning)
103
113
 
104
114
  py2docfx_logger = py2docfxLogger.get_logger(__name__)
@@ -110,19 +120,18 @@ def main(argv) -> int:
110
120
  output_doc_folder = prepare_out_dir(output_root)
111
121
 
112
122
  try:
113
- asyncio.run(donwload_package_generate_documents(
123
+ await donwload_package_generate_documents(
114
124
  package_info_list, output_root, output_doc_folder,
115
- github_token, ado_token, required_package_list))
125
+ github_token, ado_token, required_package_list)
116
126
  except Exception as e:
117
127
  msg = f"An error occurred: {e}"
118
128
  py2docfx_logger.error(msg)
129
+ fishish_up()
130
+ asyncio.get_event_loop().stop()
119
131
  raise
120
132
 
121
- warning_count, error_count = py2docfxLogger.get_warning_error_count()
122
- py2docfxLogger.output_log_by_log_level()
123
- print(f"Warning count: {warning_count}, Error count: {error_count}")
124
- logging.shutdown()
133
+ fishish_up()
125
134
  return 0
126
135
 
127
136
  if __name__ == "__main__":
128
- sys.exit(main(sys.argv[1:]))
137
+ sys.exit(asyncio.run(main(sys.argv[1:])))
@@ -22,7 +22,7 @@ PIP_INSTALL_VENV_COMMON_OPTIONS = [
22
22
  "--disable-pip-version-check",
23
23
  ]
24
24
 
25
- def install_converter_requirements(executable: str):
25
+ async def install_converter_requirements(executable: str):
26
26
  """
27
27
  Install setuptools/sphinx/pyyaml/jinja2
28
28
  Replacing logic of
@@ -41,14 +41,14 @@ def install_converter_requirements(executable: str):
41
41
  for module in REQUIREMENT_MODULES:
42
42
  msg = f"<CI INFO>: Upgrading {module}..."
43
43
  py2docfx_logger.info(msg)
44
- subprocess.run(
45
- pip_install_cmd + [module] + pip_install_common_options, check=True
46
- )
44
+ full_cmd = pip_install_cmd + [module] + pip_install_common_options
45
+ await run_async_subprocess(executable, full_cmd, py2docfx_logger)
47
46
 
48
47
  async def install_venv_requirements(venv_num: int):
49
48
  venv_exe = get_venv_exe(venv_num)
50
49
  pip_cmd = ["-m", "pip", "install", "--upgrade"]+ VENV_REQUIREMENT_MODULES
51
- await run_async_subprocess(venv_exe, pip_cmd)
50
+ py2docfx_logger = get_logger(__name__)
51
+ await run_async_subprocess(venv_exe, pip_cmd, py2docfx_logger)
52
52
 
53
53
  def get_venv_path(venv_num: int) -> str:
54
54
  return os.path.join(PACKAGE_ROOT, VENV_DIR, "venv"+str(venv_num))
@@ -85,7 +85,8 @@ def get_base_venv_sphinx_build_path() -> str:
85
85
 
86
86
  async def install_converter_requirement_async(executable: str):
87
87
  pip_cmd = PIP_INSTALL_COMMAND + PIP_INSTALL_VENV_COMMON_OPTIONS + REQUIREMENT_MODULES
88
- await run_async_subprocess(executable, pip_cmd)
88
+ py2docfx_logger = get_logger(__name__)
89
+ await run_async_subprocess(executable, pip_cmd, py2docfx_logger)
89
90
 
90
91
  async def install_required_packages(
91
92
  executable: str, required_package_list: list[PackageInfo], github_token: str, ado_token: str):
@@ -95,7 +96,8 @@ async def install_required_packages(
95
96
  # get_source(package, idx, vststoken=ado_token, githubtoken=github_token)
96
97
  package_name, options = package.get_install_command()
97
98
  pip_cmd = PIP_INSTALL_COMMAND + PIP_INSTALL_VENV_COMMON_OPTIONS + options + [package_name]
98
- await run_async_subprocess(executable, pip_cmd)
99
+ py2docfx_logger = get_logger(__name__)
100
+ await run_async_subprocess(executable, pip_cmd, py2docfx_logger)
99
101
 
100
102
  async def create_environment(venv_path: int):
101
103
  if os.name == 'nt':
@@ -125,7 +127,7 @@ async def prepare_venv(venv_num: int, package_info: PackageInfo, package_number:
125
127
  py2docfx_logger = get_logger(__name__)
126
128
  await create_environment(get_venv_path(venv_num))
127
129
  await install_venv_requirements(venv_num)
128
- get_source(get_venv_exe(venv_num), package_info, package_number, vststoken=ado_token, githubtoken=github_token)
130
+ await get_source(get_venv_exe(venv_num), package_info, package_number, vststoken=ado_token, githubtoken=github_token)
129
131
  package_name, options = package_info.get_install_command()
130
132
  await pip_utils.install_in_exe_async(get_venv_exe(venv_num), package_name, options)
131
133
  msg = f"<CI INFO>: venv{venv_num} setup complete."
@@ -138,8 +140,9 @@ async def remove_environment(venv_num: int):
138
140
  msg = f"<CI INFO>: Removing venv{venv_num}..."
139
141
  py2docfx_logger.info(msg)
140
142
  # Create a subprocess to run the shell command for removing the directory
143
+ cmd = f'rm -rf {venv_path}' if os.name != 'nt' else f'rmdir /S /Q {venv_path}'
141
144
  process = await asyncio.create_subprocess_shell(
142
- f'rm -rf {venv_path}' if os.name != 'nt' else f'rmdir /S /Q {venv_path}',
145
+ cmd,
143
146
  stdout=asyncio.subprocess.PIPE,
144
147
  stderr=asyncio.subprocess.PIPE
145
148
  )
@@ -150,4 +153,4 @@ async def remove_environment(venv_num: int):
150
153
  else:
151
154
  msg = f"<CI ERROR>: Failed to remove venv{venv_num}. Error: {stderr.decode()}"
152
155
  py2docfx_logger.error(msg)
153
- raise RuntimeError()
156
+ raise subprocess.CalledProcessError(process.returncode, cmd, stdout, stderr)
@@ -10,8 +10,8 @@ from py2docfx.convert_prepare.subpackage import merge_subpackage_files
10
10
 
11
11
  CONF_TEMPLATE_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "conf_templates")
12
12
 
13
- def generate_document(pkg: PackageInfo, output_root: str | os.PathLike, sphinx_build_path: str, extra_package_path: str, executable=sys.executable):
14
- py2docfx_logger = get_package_logger(__name__)
13
+ async def generate_document(pkg: PackageInfo, output_root: str | os.PathLike, sphinx_build_path: str, extra_package_path: str, executable=sys.executable):
14
+ py2docfx_logger = get_package_logger(__name__, pkg.name)
15
15
  # Copy manual written RST from target doc repo
16
16
  package_paths = pkg.path
17
17
  if output_root:
@@ -28,12 +28,12 @@ def generate_document(pkg: PackageInfo, output_root: str | os.PathLike, sphinx_b
28
28
  exclude_paths = pkg.get_exluded_command()
29
29
 
30
30
  if pkg.install_type == pkg.InstallType.SOURCE_CODE and getattr(pkg, "branch", None):
31
- checkout(package_paths.source_folder, pkg.branch)
31
+ await checkout(package_paths.source_folder, pkg.branch)
32
32
 
33
33
  msg = f"<CI INFO>: Generating RST files for {pkg.name}."
34
34
  py2docfx_logger.info(msg)
35
35
 
36
- subpackages_rst_record = run_apidoc(package_paths.doc_folder, package_paths.source_folder,
36
+ subpackages_rst_record = await run_apidoc(pkg.name, package_paths.doc_folder, package_paths.source_folder,
37
37
  exclude_paths, pkg)
38
38
 
39
39
  msg = f"<CI INFO>: Listing RST files:"
@@ -45,7 +45,7 @@ def generate_document(pkg: PackageInfo, output_root: str | os.PathLike, sphinx_b
45
45
  py2docfx_logger.info(msg)
46
46
 
47
47
  generate_conf(pkg, package_paths.doc_folder, CONF_TEMPLATE_DIR)
48
- run_converter(package_paths.doc_folder, package_paths.yaml_output_folder, sphinx_build_path, extra_package_path, executable=executable)
48
+ await run_converter(pkg.name, package_paths.doc_folder, package_paths.yaml_output_folder, sphinx_build_path, extra_package_path, executable=executable)
49
49
 
50
50
  subpackages_path_record = {}
51
51
  if pkg.build_in_subpackage:
@@ -53,6 +53,6 @@ def generate_document(pkg: PackageInfo, output_root: str | os.PathLike, sphinx_b
53
53
  for (subpackage_name, subpackage_path) in subpackages_rst_record.items():
54
54
  subpackage_yaml_path = os.path.join(subpackages_yaml_path, subpackage_name)
55
55
  subpackages_path_record[subpackage_name] = subpackage_yaml_path
56
- run_converter(subpackage_path, subpackage_yaml_path, sphinx_build_path, extra_package_path, executable=executable)
56
+ await run_converter(pkg.name, subpackage_path, subpackage_yaml_path, sphinx_build_path, extra_package_path, executable=executable)
57
57
 
58
58
  merge_subpackage_files(subpackages_path_record, package_paths.yaml_output_folder, pkg.name)
@@ -72,7 +72,7 @@ def update_package_info(executable: str, pkg: PackageInfo, source_folder: str):
72
72
  source_folder=source_folder, yaml_output_folder=yaml_output_folder, package_name=pkg.name
73
73
  )
74
74
 
75
- def get_source(executable: str, pkg: PackageInfo, cnt: int, vststoken=None, githubtoken=None):
75
+ async def get_source(executable: str, pkg: PackageInfo, cnt: int, vststoken=None, githubtoken=None):
76
76
  py2docfx_logger = get_logger(__name__)
77
77
  path_cnt = str(cnt)
78
78
  dist_dir = path.join(DIST_TEMP, path_cnt)
@@ -81,7 +81,7 @@ def get_source(executable: str, pkg: PackageInfo, cnt: int, vststoken=None, gith
81
81
  if pkg.url:
82
82
  repo_folder = path.join(SOURCE_REPO, path_cnt)
83
83
  token = githubtoken if "github.com" in pkg.url else vststoken
84
- source_folder = git.clone(
84
+ source_folder = await git.clone(
85
85
  repo_location=pkg.url,
86
86
  branch=pkg.branch,
87
87
  folder=repo_folder,
@@ -94,7 +94,7 @@ def get_source(executable: str, pkg: PackageInfo, cnt: int, vststoken=None, gith
94
94
  sys.path.insert(0, source_folder)
95
95
  elif pkg.install_type == PackageInfo.InstallType.PYPI:
96
96
  full_name = pkg.get_combined_name_version()
97
- pip_utils.download(
97
+ await pip_utils.download(
98
98
  full_name,
99
99
  dist_dir,
100
100
  extra_index_url=pkg.extra_index_url,
@@ -102,17 +102,17 @@ def get_source(executable: str, pkg: PackageInfo, cnt: int, vststoken=None, gith
102
102
  )
103
103
  # unpack the downloaded wheel file.
104
104
  downloaded_dist_file = path.join(dist_dir, os.listdir(dist_dir)[0])
105
- pack.unpack_dist(downloaded_dist_file)
105
+ await pack.unpack_dist(pkg.name, downloaded_dist_file)
106
106
  os.remove(downloaded_dist_file)
107
107
  source_folder = path.join(
108
108
  path.dirname(downloaded_dist_file),
109
109
  os.listdir(dist_dir)[0]
110
110
  )
111
111
  elif pkg.install_type == PackageInfo.InstallType.DIST_FILE:
112
- pip_utils.download(pkg.location, dist_dir, prefer_source_distribution=False)
112
+ await pip_utils.download(pkg.location, dist_dir, prefer_source_distribution=False)
113
113
  # unpack the downloaded dist file.
114
114
  downloaded_dist_file = path.join(dist_dir, os.listdir(dist_dir)[0])
115
- pack.unpack_dist(downloaded_dist_file)
115
+ await pack.unpack_dist(pkg.name, downloaded_dist_file)
116
116
  os.remove(downloaded_dist_file)
117
117
  if downloaded_dist_file.endswith(".tar.gz"):
118
118
  downloaded_dist_file = downloaded_dist_file.rsplit(".", maxsplit=1)[
@@ -124,6 +124,6 @@ def get_source(executable: str, pkg: PackageInfo, cnt: int, vststoken=None, gith
124
124
  else:
125
125
  msg = f"Unknown install type: {pkg.install_type}"
126
126
  py2docfx_logger.error(msg)
127
- raise ValueError()
127
+ raise ValueError(msg)
128
128
 
129
129
  update_package_info(executable, pkg, source_folder)
@@ -1,11 +1,11 @@
1
1
  import re
2
2
  import subprocess
3
3
 
4
- from py2docfx.docfx_yaml.logger import get_logger, log_git_clone_subprocess_ouput
4
+ from py2docfx.docfx_yaml.logger import get_logger, run_async_subprocess_without_executable
5
5
 
6
6
  repoMap = {}
7
7
 
8
- def clone(repo_location, branch, folder, extra_token=None):
8
+ async def clone(repo_location, branch, folder, extra_token=None):
9
9
  """
10
10
  Clone a git repo to the folder
11
11
  When extra_token isn't None, add it to the command
@@ -22,7 +22,7 @@ def clone(repo_location, branch, folder, extra_token=None):
22
22
  if not test_url(repo_location, extra_token):
23
23
  msg = "Git repo address {} is not a valid URL.".format(repo_location)
24
24
  py2docfx_logger.error(msg)
25
- raise ValueError()
25
+ raise ValueError(msg)
26
26
  else:
27
27
  # Remove http(s):// from url to record. Further avoid dup-clone.
28
28
  pureURL = re.sub("^\s*https?://", "", repo_location)
@@ -57,8 +57,7 @@ def clone(repo_location, branch, folder, extra_token=None):
57
57
  extra_token
58
58
  ),
59
59
  ]
60
- output = subprocess.run(clone_params, check=True, capture_output=True, text=True)
61
- log_git_clone_subprocess_ouput(output, py2docfx_logger)
60
+ await run_async_subprocess_without_executable(clone_params, py2docfx_logger)
62
61
  repoMap[pureURL] = folder
63
62
  msg = "<CI INFO>: Repo {} successfully cloned in {}...".format(
64
63
  repo_location, repoMap[pureURL]
@@ -123,7 +122,7 @@ def convertBranch(repo_url, branch, extraHeader):
123
122
  return result
124
123
 
125
124
 
126
- def checkout(folder, branch):
125
+ async def checkout(folder, branch):
127
126
  """
128
127
  Checkout one folder to a given branch
129
128
 
@@ -145,8 +144,7 @@ def checkout(folder, branch):
145
144
  if "* {}".format(branch) not in branches:
146
145
  # branch is not exactly current branch
147
146
  if branch not in branches:
148
- output = subprocess.run(
149
- [
147
+ fetch_cmd = [
150
148
  "git",
151
149
  "-C",
152
150
  folder,
@@ -154,12 +152,11 @@ def checkout(folder, branch):
154
152
  "--quiet",
155
153
  remote,
156
154
  "{}:{}".format(branch, branch),
157
- ], capture_output=True, text=True
158
- )
159
- log_subprocess_ouput(output, py2docfx_logger)
155
+ ]
156
+ await run_async_subprocess_without_executable(fetch_cmd, py2docfx_logger)
160
157
 
161
- output = subprocess.run(["git", "-C", folder, "checkout", "--quiet", branch], check=True, capture_output=True, text=True)
162
- log_subprocess_ouput(output, py2docfx_logger)
158
+ checkout_cmd = ["git", "-C", folder, "checkout", "--quiet", branch]
159
+ await run_async_subprocess_without_executable(checkout_cmd, py2docfx_logger)
163
160
  msg = "<CI INFO>: Switched to branch {}.".format(branch)
164
161
  py2docfx_logger.info(msg)
165
162
 
@@ -1,6 +1,6 @@
1
1
  from py2docfx.convert_prepare.package_info import PackageInfo
2
2
  import py2docfx.convert_prepare.pip_utils as pip_utils
3
3
 
4
- def install_package(pkg: PackageInfo):
4
+ async def install_package(pkg: PackageInfo):
5
5
  package_name, options = pkg.get_install_command()
6
- pip_utils.install(package_name, options)
6
+ await pip_utils.install(package_name, options)
@@ -1,14 +1,13 @@
1
1
  import shutil
2
2
  from os import path
3
3
  import os
4
- import subprocess
5
4
  import sys
6
5
 
7
- from py2docfx.docfx_yaml.logger import get_package_logger, log_subprocess_ouput
6
+ from py2docfx.docfx_yaml.logger import get_package_logger, run_async_subprocess
8
7
 
9
- def unpack_dist(dist_file):
8
+ async def unpack_dist(package_name, dist_file):
10
9
  if dist_file.endswith(".whl"):
11
- unpack_wheel(dist_file)
10
+ await unpack_wheel(package_name, dist_file)
12
11
  else:
13
12
  unpack_compressed(dist_file)
14
13
 
@@ -20,20 +19,18 @@ def unpack_compressed(file_path):
20
19
  shutil.unpack_archive(file_path, path.dirname(file_path))
21
20
 
22
21
 
23
- def unpack_wheel(file_path):
22
+ async def unpack_wheel(package_name, file_path):
24
23
  """
25
24
  Transform a wheel file to a folder containing source code
26
25
  """
27
- py2docfx_logger = get_package_logger(__name__)
28
- command = [sys.executable,
29
- '-m',
26
+ py2docfx_logger = get_package_logger(__name__, package_name)
27
+ command = ['-m',
30
28
  'wheel',
31
29
  'unpack',
32
30
  file_path,
33
31
  '-d',
34
32
  os.path.dirname(file_path)]
35
- output = subprocess.run(command, check=True, capture_output=True, text=True)
36
- log_subprocess_ouput(output, py2docfx_logger)
33
+ await run_async_subprocess(sys.executable, command, py2docfx_logger)
37
34
  extract_folder = [file for file in os.listdir(path.dirname(file_path)) if path.isdir(path.join(path.dirname(file_path), file))][0]
38
35
  data_folder = path.join(path.dirname(file_path), extract_folder, extract_folder + ".data")
39
36
  if path.exists(data_folder):
@@ -26,7 +26,7 @@ class PackageInfo:
26
26
  name, value
27
27
  )
28
28
  py2docfx_logger.error(message)
29
- raise ValueError()
29
+ raise ValueError(message)
30
30
 
31
31
  @classmethod
32
32
  def parse_from(cls, dict, reading_required_packages=False):
@@ -64,7 +64,7 @@ class PackageInfo:
64
64
  if not package_info.folder:
65
65
  msg = "When install_type is source_code, url or folder should be provided"
66
66
  py2docfx_logger.error(msg)
67
- raise ValueError()
67
+ raise ValueError(msg)
68
68
  else:
69
69
  msg = f'Read source code from local folder: {package_info.folder}'
70
70
  py2docfx_logger.info(msg)
@@ -149,7 +149,7 @@ class PackageInfo:
149
149
  else:
150
150
  msg = "Should set source code location before build documents"
151
151
  py2docfx_logger.error(msg)
152
- raise ValueError()
152
+ raise ValueError(msg)
153
153
  exclude_path = []
154
154
  if code_location:
155
155
  exclude_path.append(os.path.join(code_location, "build/*"))
@@ -1,7 +1,7 @@
1
- import subprocess
1
+ import sys
2
2
 
3
3
  from py2docfx import PACKAGE_ROOT
4
- from py2docfx.docfx_yaml.logger import get_logger, log_subprocess_ouput, run_async_subprocess
4
+ from py2docfx.docfx_yaml.logger import get_logger, run_async_subprocess, run_async_subprocess_without_executable
5
5
 
6
6
  PYPI = "pypi"
7
7
 
@@ -14,7 +14,7 @@ pip_install_common_options = [
14
14
  "--verbose"
15
15
  ]
16
16
 
17
- def download(package_name, path, extra_index_url=None, prefer_source_distribution=True):
17
+ async def download(package_name, path, extra_index_url=None, prefer_source_distribution=True):
18
18
  # Downloads a package from PyPI to the specified path using pip.
19
19
  download_param = ["pip", "download", "--dest", path, "--no-deps", package_name]
20
20
  if extra_index_url:
@@ -24,29 +24,27 @@ def download(package_name, path, extra_index_url=None, prefer_source_distributio
24
24
  download_param.append("--no-binary=:all:")
25
25
  else:
26
26
  download_param.append("--prefer-binary")
27
- output = subprocess.run(download_param, check=True, cwd=PACKAGE_ROOT, capture_output=True, text=True)
28
- py2docfx_logger = get_logger(__name__)
29
- log_subprocess_ouput(output, py2docfx_logger)
30
27
 
28
+ py2docfx_logger = get_logger(__name__)
29
+ await run_async_subprocess_without_executable(download_param, py2docfx_logger, cwd=PACKAGE_ROOT)
31
30
 
32
- def install(package_name, options):
31
+ async def install(package_name, options):
33
32
  # Installs a package from PyPI using pip.
34
33
  install_param = "pip install {} {}".format(
35
34
  " ".join(pip_install_common_options + options), package_name
36
35
  ).split(" ")
37
- output = subprocess.run(install_param, check=True, cwd=PACKAGE_ROOT, capture_output=True, text=True)
38
36
  py2docfx_logger = get_logger(__name__)
39
- log_subprocess_ouput(output, py2docfx_logger)
37
+ await run_async_subprocess_without_executable(install_param, py2docfx_logger, cwd=PACKAGE_ROOT)
40
38
 
41
- def install_in_exe(exe_path, package_name, options):
39
+ async def install_in_exe(exe_path, package_name, options):
42
40
  # Installs a package from PyPI using pip.
43
41
  install_param = [exe_path] + "-m pip install {} {}".format(
44
42
  " ".join(pip_install_common_options + options), package_name
45
43
  ).split(" ")
46
- output = subprocess.run(install_param, check=True, cwd=PACKAGE_ROOT, capture_output=True, text=True)
47
44
  py2docfx_logger = get_logger(__name__)
48
- log_subprocess_ouput(output, py2docfx_logger)
49
-
45
+ await run_async_subprocess(exe_path, install_param, py2docfx_logger, cwd=PACKAGE_ROOT)
46
+
50
47
  async def install_in_exe_async(exe_path, package_name, options):
51
48
  pip_cmd = ["-m", "pip", "install"]+ pip_install_common_options + options + [package_name]
52
- await run_async_subprocess(exe_path, pip_cmd)
49
+ py2docfx_logger = get_logger(__name__)
50
+ await run_async_subprocess(exe_path, pip_cmd, py2docfx_logger)
@@ -21,8 +21,9 @@ def merge_toc(
21
21
  if not toc_content.endswith("\n"):
22
22
  root_toc_handle.write("\n")
23
23
  else:
24
- py2docfx_logger.error(f"TOC content empty: {package_toc_path}")
25
- raise ValueError()
24
+ msg = f"TOC content empty: {package_toc_path}"
25
+ py2docfx_logger.error(msg)
26
+ raise ValueError(msg)
26
27
 
27
28
  # delete package toc.yml
28
29
  os.remove(package_toc_path)
@@ -1,9 +1,10 @@
1
+ import asyncio
1
2
  import os
2
- import sys
3
3
  import subprocess
4
+ import sys
4
5
 
5
6
  from py2docfx import PACKAGE_ROOT
6
- from py2docfx.docfx_yaml.logger import get_package_logger,log_subprocess_ouput
7
+ from py2docfx.docfx_yaml.logger import get_package_logger,run_async_subprocess
7
8
  from py2docfx.convert_prepare.package_info import PackageInfo
8
9
  from py2docfx.convert_prepare.paths import folder_is_hidden
9
10
  from py2docfx.convert_prepare.subpackage import (get_subpackages,
@@ -11,14 +12,14 @@ from py2docfx.convert_prepare.subpackage import (get_subpackages,
11
12
 
12
13
  DEBUG_SPHINX_FLAG = 'PY2DOCFX_DEBUG_SPHINX'
13
14
 
14
- def run_apidoc(rst_path, source_code_path, exclude_paths, package_info: PackageInfo):
15
+ async def run_apidoc(package_name, rst_path, source_code_path, exclude_paths, package_info: PackageInfo):
15
16
  """
16
17
  Run sphinx-apidoc to generate RST inside rst_path folder
17
18
 
18
19
  Replacing
19
20
  https://apidrop.visualstudio.com/Content%20CI/_git/ReferenceAutomation?path=/Python/build.ps1&line=110&lineEnd=126&lineStartColumn=1&lineEndColumn=14&lineStyle=plain&_a=contents
20
21
  """
21
- py2docfx_logger = get_package_logger(__name__)
22
+ py2docfx_logger = get_package_logger(__name__, package_name)
22
23
  subfolderList = [name for name in
23
24
  os.listdir(source_code_path)
24
25
  if os.path.isdir(os.path.join(source_code_path, name))
@@ -40,8 +41,8 @@ def run_apidoc(rst_path, source_code_path, exclude_paths, package_info: PackageI
40
41
  subfolderPath,
41
42
  ]
42
43
  args.extend(exclude_paths)
43
- output = subprocess.run([sys.executable, "-m", "sphinx.ext.apidoc"] + args, capture_output=True, text=True)
44
- log_subprocess_ouput(output, py2docfx_logger)
44
+ full_args = ["-m", "sphinx.ext.apidoc"] + args
45
+ await run_async_subprocess(sys.executable, full_args, py2docfx_logger)
45
46
  if package_info.build_in_subpackage and subfolder == "azure":
46
47
  subpackages_rst_record = move_rst_files_to_subfolder(
47
48
  package_paths.doc_folder, package_info.name,
@@ -49,7 +50,13 @@ def run_apidoc(rst_path, source_code_path, exclude_paths, package_info: PackageI
49
50
  return subpackages_rst_record
50
51
 
51
52
 
52
- def run_converter(rst_path, out_path, sphinx_build_path: str, extra_package_path: str, conf_path = None, executable = sys.executable):
53
+ async def run_converter(package_name: str,
54
+ rst_path,
55
+ out_path,
56
+ sphinx_build_path: str,
57
+ extra_package_path: str,
58
+ conf_path = None,
59
+ executable = sys.executable):
53
60
  """
54
61
  Take rst files as input and run sphinx converter
55
62
 
@@ -58,7 +65,7 @@ def run_converter(rst_path, out_path, sphinx_build_path: str, extra_package_path
58
65
  Replacing
59
66
  https://apidrop.visualstudio.com/Content%20CI/_git/ReferenceAutomation?path=/Python/build.ps1&line=150&lineEnd=161&lineStartColumn=13&lineEndColumn=52&lineStyle=plain&_a=contents
60
67
  """
61
- py2docfx_logger = get_package_logger(__name__)
68
+ py2docfx_logger = get_package_logger(__name__, package_name)
62
69
  outdir = os.path.join(out_path, "_build")
63
70
 
64
71
  # Sphinx/docutils have memory leak including linecaches, module-import-caches,
@@ -67,9 +74,8 @@ def run_converter(rst_path, out_path, sphinx_build_path: str, extra_package_path
67
74
  if not sys.executable:
68
75
  msg = "Can't get the executable binary for the Python interpreter."
69
76
  py2docfx_logger.error(msg)
70
- raise ValueError()
77
+ raise ValueError(msg)
71
78
  sphinx_param = [
72
- executable,
73
79
  sphinx_build_path,
74
80
  rst_path,
75
81
  outdir,
@@ -86,6 +92,22 @@ def run_converter(rst_path, out_path, sphinx_build_path: str, extra_package_path
86
92
  env_tmp["PYTHONPATH"] = f"{extra_package_path};{package_root_parent};"
87
93
  else:
88
94
  env_tmp["PYTHONPATH"] = f"{extra_package_path}:{package_root_parent}:"
89
- output = subprocess.run(sphinx_param, check=True, cwd=PACKAGE_ROOT, env=env_tmp, capture_output=True ,text=True)
90
- log_subprocess_ouput(output, py2docfx_logger)
95
+ os.environ['PROCESSING_PACKAGE_NAME'] = package_name
96
+ proc = await asyncio.create_subprocess_exec(
97
+ executable, *sphinx_param,
98
+ cwd=PACKAGE_ROOT,
99
+ env=env_tmp,
100
+ stdout=asyncio.subprocess.PIPE,
101
+ stderr=asyncio.subprocess.PIPE
102
+ )
103
+ stdout, stderr = await proc.communicate()
104
+ return_code = proc.returncode
105
+
106
+ if return_code == 0:
107
+ py2docfx_logger.info(f"{stdout}")
108
+ py2docfx_logger.info(f"{stderr}")
109
+ else:
110
+ py2docfx_logger.error(f"{stderr}")
111
+ raise subprocess.CalledProcessError(return_code, sphinx_param, stdout, stderr)
112
+
91
113
  return outdir
@@ -1,8 +1,6 @@
1
- import asyncio
2
1
  import os
3
2
  import shutil
4
3
  import pytest
5
- import sys
6
4
 
7
5
  from py2docfx.convert_prepare.environment import create_environment
8
6
  from py2docfx.convert_prepare.environment import remove_environment
@@ -14,7 +12,6 @@ from py2docfx.convert_prepare.environment import prepare_base_venv
14
12
  from py2docfx.convert_prepare.environment import get_base_venv_exe
15
13
  from py2docfx.convert_prepare.environment import get_base_venv_path
16
14
  from py2docfx.convert_prepare.package_info import PackageInfo
17
- pytest_plugins = ('pytest_asyncio',)
18
15
 
19
16
  @pytest.mark.asyncio
20
17
  async def test_venv_creation_package_install_remove():
@@ -4,6 +4,7 @@ Test the generate_document function.
4
4
  import os
5
5
  import sys
6
6
  import shutil
7
+ import pytest
7
8
  import sphinx
8
9
  import sphinx.cmd.build
9
10
 
@@ -11,7 +12,8 @@ from py2docfx.convert_prepare.generate_document import generate_document
11
12
  from py2docfx.convert_prepare.package_info import PackageInfo
12
13
  from py2docfx.convert_prepare.source import Source
13
14
 
14
- def test_generate_document(tmp_path):
15
+ @pytest.mark.asyncio
16
+ async def test_generate_document(tmp_path):
15
17
  """
16
18
  Test the generate_document function.
17
19
  """
@@ -42,7 +44,7 @@ def test_generate_document(tmp_path):
42
44
 
43
45
  # call the function
44
46
 
45
- generate_document(package, output_root, sphinx_build_path = sphinx.cmd.build.__file__, extra_package_path = source_folder)
47
+ await generate_document(package, output_root, sphinx_build_path = sphinx.cmd.build.__file__, extra_package_path = source_folder)
46
48
 
47
49
  #assert the result
48
50
  yaml_path = os.path.join(yaml_output_folder, "_build", "docfx_yaml")
@@ -6,6 +6,7 @@ import shutil
6
6
  import stat
7
7
  import pytest
8
8
  import sys
9
+
9
10
  from py2docfx.convert_prepare import git
10
11
  from py2docfx.convert_prepare import get_source
11
12
  from py2docfx.convert_prepare.package_info import PackageInfo
@@ -61,7 +62,8 @@ def test_update_package_info(init_package_info):
61
62
  assert package.name == "mock_package"
62
63
  assert package.version == "1.2.0"
63
64
 
64
- def test_get_source_git_clone(init_package_info):
65
+ @pytest.mark.asyncio
66
+ async def test_get_source_git_clone(init_package_info):
65
67
  """
66
68
  Test the git clone of get_source
67
69
  """
@@ -76,10 +78,11 @@ def test_get_source_git_clone(init_package_info):
76
78
  package.branch = "main"
77
79
  package.folder = None
78
80
  package.url = "https://github.com/Azure/azure-iot-hub-python"
79
- get_source.get_source(sys.executable, package, 0)
81
+ await get_source.get_source(sys.executable, package, 0)
80
82
  assert git.status("source_repo/0") is True
81
83
 
82
- def test_get_source_dist_file_zip(init_package_info):
84
+ @pytest.mark.asyncio
85
+ async def test_get_source_dist_file_zip(init_package_info):
83
86
  """
84
87
  Test the zip dist file download of get_source
85
88
  """
@@ -92,12 +95,13 @@ def test_get_source_dist_file_zip(init_package_info):
92
95
  package.extra_index_url = None
93
96
  package.prefer_source_distribution = True
94
97
  package.location = "https://files.pythonhosted.org/packages/3e/71/f6f71a276e2e69264a97ad39ef850dca0a04fce67b12570730cb38d0ccac/azure-common-1.1.28.zip"
95
- get_source.get_source(sys.executable, package, 1)
98
+ await get_source.get_source(sys.executable, package, 1)
96
99
  assert os.path.exists("dist_temp/1/azure-common-1.1.28")
97
100
  assert package.path.source_folder == os.path.join("dist_temp", "1", "azure-common-1.1.28")
98
101
  assert os.path.exists("dist_temp/1/azure-common-1.1.28.zip") is False
99
102
 
100
- def test_get_source_dist_file_whl(init_package_info):
103
+ @pytest.mark.asyncio
104
+ async def test_get_source_dist_file_whl(init_package_info):
101
105
  """
102
106
  Test the whl dist file download of get_source
103
107
  """
@@ -110,12 +114,13 @@ def test_get_source_dist_file_whl(init_package_info):
110
114
  package.extra_index_url = None
111
115
  package.prefer_source_distribution = True
112
116
  package.location = "https://files.pythonhosted.org/packages/62/55/7f118b9c1b23ec15ca05d15a578d8207aa1706bc6f7c87218efffbbf875d/azure_common-1.1.28-py2.py3-none-any.whl"
113
- get_source.get_source(sys.executable, package, 2)
117
+ await get_source.get_source(sys.executable, package, 2)
114
118
  assert os.path.exists("dist_temp/2/azure_common-1.1.28")
115
119
  assert package.path.source_folder == os.path.join("dist_temp", "2", "azure_common-1.1.28")
116
120
  assert os.path.exists("dist_temp/2/azure_common-1.1.28-py2.py3-none-any.whl") is False
117
121
 
118
- def test_get_source_dist_file_tar(init_package_info):
122
+ @pytest.mark.asyncio
123
+ async def test_get_source_dist_file_tar(init_package_info):
119
124
  """
120
125
  Test the tar dist file download of get_source
121
126
  """
@@ -128,12 +133,13 @@ def test_get_source_dist_file_tar(init_package_info):
128
133
  package.extra_index_url = None
129
134
  package.prefer_source_distribution = True
130
135
  package.location = "https://files.pythonhosted.org/packages/fa/19/43a9eb812b4d6071fdc2c55640318f7eb5a1be8dbd3b6f9d96a1996e1bb6/azure-core-1.29.4.tar.gz"
131
- get_source.get_source(sys.executable, package, 3)
136
+ await get_source.get_source(sys.executable, package, 3)
132
137
  assert os.path.exists("dist_temp/3/azure-core-1.29.4")
133
138
  assert package.path.source_folder == os.path.join("dist_temp", "3", "azure-core-1.29.4")
134
139
  assert os.path.exists("dist_temp/3/azure-core-1.29.4.tar.gz") is False
135
140
 
136
- def test_get_source_pip_whl(init_package_info):
141
+ @pytest.mark.asyncio
142
+ async def test_get_source_pip_whl(init_package_info):
137
143
  """
138
144
  Test the pip install of get_source with prefer_source_distribution = False
139
145
  """
@@ -145,12 +151,13 @@ def test_get_source_pip_whl(init_package_info):
145
151
  package.build_in_subpackage = False
146
152
  package.extra_index_url = None
147
153
  package.prefer_source_distribution = False
148
- get_source.get_source(sys.executable, package, 4)
154
+ await get_source.get_source(sys.executable, package, 4)
149
155
  assert os.path.exists("dist_temp/4/azure_common-1.1.28")
150
156
  assert package.path.source_folder == os.path.join("dist_temp", "4", "azure_common-1.1.28")
151
157
  assert os.path.exists("dist_temp/4/azure_common-1.1.28-py2.py3-none-any.whl") is False
152
158
 
153
- def test_get_source_pip_zip(init_package_info):
159
+ @pytest.mark.asyncio
160
+ async def test_get_source_pip_zip(init_package_info):
154
161
  """
155
162
  Test the pip install of get_source with prefer_source_distribution = True
156
163
  """
@@ -162,12 +169,13 @@ def test_get_source_pip_zip(init_package_info):
162
169
  package.build_in_subpackage = False
163
170
  package.extra_index_url = None
164
171
  package.prefer_source_distribution = True
165
- get_source.get_source(sys.executable, package, 5)
172
+ await get_source.get_source(sys.executable, package, 5)
166
173
  assert os.path.exists("dist_temp/5/azure-common-1.1.28")
167
174
  assert package.path.source_folder == os.path.join("dist_temp", "5", "azure-common-1.1.28")
168
175
  assert os.path.exists("dist_temp/5/azure-common-1.1.28.zip") is False
169
176
 
170
- def test_get_source_zip_file_at_position_0(init_package_info):
177
+ @pytest.mark.asyncio
178
+ async def test_get_source_zip_file_at_position_0(init_package_info):
171
179
  """
172
180
  Test the pip install of packages with zip or tar file at position 0 in the dirctory list
173
181
  """
@@ -179,7 +187,7 @@ def test_get_source_zip_file_at_position_0(init_package_info):
179
187
  package.build_in_subpackage = False
180
188
  package.extra_index_url = None
181
189
  package.prefer_source_distribution = True
182
- get_source.get_source(sys.executable, package, 6)
190
+ await get_source.get_source(sys.executable, package, 6)
183
191
  assert os.path.exists("dist_temp/6/azure_template-0.1.0b3942895")
184
192
  assert package.path.source_folder == os.path.join("dist_temp", "6", "azure_template-0.1.0b3942895")
185
193
  assert os.path.exists("dist_temp/6/azure_template-0.1.0b3942895.tar.gz") is False
@@ -5,6 +5,8 @@ import subprocess
5
5
  import shutil
6
6
  import glob
7
7
  from os import path
8
+ import pytest
9
+
8
10
  from py2docfx.convert_prepare.pack import unpack_compressed, unpack_wheel
9
11
 
10
12
  SRC_DIR = path.abspath("convert_prepare/tests/data/pack/")
@@ -81,8 +83,8 @@ def test_pack_unpack_compressed(tmp_path):
81
83
  path.abspath("convert_prepare/tests/data/pack"), tmp_path / "gz", [], [str(gz_file_path)]
82
84
  )
83
85
 
84
-
85
- def test_pack_unpack_wheel(tmp_path):
86
+ @pytest.mark.asyncio
87
+ async def test_pack_unpack_wheel(tmp_path):
86
88
  def _prepare_wheel(target_path):
87
89
  subprocess.run(
88
90
  ["pip", "wheel", ".", "--wheel-dir", str(target_path / "wheel")],
@@ -107,7 +109,8 @@ def test_pack_unpack_wheel(tmp_path):
107
109
  wheel_path = path.join(tmp_path / "wheel", wheel_name)
108
110
 
109
111
  # unpack and assert the file list
110
- unpack_wheel(wheel_path)
112
+ package_name = wheel_name.split("-")[0]
113
+ await unpack_wheel(package_name, wheel_path)
111
114
  _assert_file_list_same(
112
115
  path.abspath("convert_prepare/tests/data/pack"),
113
116
  tmp_path / "wheel" / "foo-0.1",
@@ -1,4 +1,3 @@
1
- import pytest
2
1
  import os
3
2
 
4
3
  from py2docfx.convert_prepare.params import load_file_params, load_command_params
@@ -1,5 +1,5 @@
1
- import pytest
2
1
  import os
2
+ import pytest
3
3
  import shutil
4
4
  import sphinx
5
5
  import sphinx.cmd.build
@@ -24,20 +24,22 @@ def init_paths(tmp_path):
24
24
  package_info.name = 'testcode'
25
25
  return rst_path, destination
26
26
 
27
-
28
- def test_run_apidoc(tmp_path):
27
+ @pytest.mark.asyncio
28
+ async def test_run_apidoc(tmp_path):
29
29
  rst_path, source_code_path = init_paths(tmp_path)
30
- run_apidoc(rst_path, source_code_path, package_info.get_exluded_command(), package_info)
30
+ package_name = "testcode"
31
+ await run_apidoc(package_name, rst_path, source_code_path, package_info.get_exluded_command(), package_info)
31
32
 
32
33
  # List all files under rst_path
33
34
  rst_list = os.listdir(rst_path)
34
35
  assert "testcode.fakemodule.rst" in rst_list
35
36
  assert "testcode.exclude.rst" not in rst_list
36
37
 
37
-
38
- def test_run_converter(tmp_path):
38
+ @pytest.mark.asyncio
39
+ async def test_run_converter(tmp_path):
39
40
  rst_path, source_code_path = init_paths(tmp_path)
40
- run_apidoc(rst_path, source_code_path, package_info.get_exluded_command(), package_info)
41
+ package_name = "testcode"
42
+ await run_apidoc(package_name, rst_path, source_code_path, package_info.get_exluded_command(), package_info)
41
43
 
42
44
  # prepare conf.py, index.rst and docfx_yaml
43
45
  conf_path = os.path.abspath("convert_prepare/tests/data/sphinx_caller/conf.py")
@@ -49,7 +51,7 @@ def test_run_converter(tmp_path):
49
51
  index_rst.write("")
50
52
 
51
53
  out_path = os.path.join(tmp_path, "out")
52
- out_path = run_converter(rst_path, out_path, sphinx_build_path = sphinx.cmd.build.__file__, extra_package_path = source_code_path, conf_path=rst_path)
54
+ out_path = await run_converter(package_name, rst_path, out_path, sphinx_build_path = sphinx.cmd.build.__file__, extra_package_path = source_code_path, conf_path=rst_path)
53
55
 
54
56
  if os.path.exists(out_path):
55
57
  yaml_list = os.listdir(os.path.join(out_path, "docfx_yaml"))
@@ -1,5 +1,6 @@
1
1
  import shutil, os, yaml
2
2
  from os import path
3
+
3
4
  from py2docfx.convert_prepare.subpackage import (
4
5
  get_subpackages,
5
6
  move_rst_files_to_subfolder,
@@ -315,7 +315,7 @@ def build_finished(app, exception):
315
315
  if len(toc_yaml) == 0:
316
316
  msg = "No documentation for this module."
317
317
  py2docfx_logger.error(msg)
318
- raise RuntimeError()
318
+ raise RuntimeError(msg)
319
319
 
320
320
  toc_file = os.path.join(normalized_outdir, 'toc.yml')
321
321
  with open(toc_file, 'w') as writable:
@@ -34,8 +34,13 @@ def check_log_file_exists(log_file_path):
34
34
  with open(log_file_path, 'w') as f:
35
35
  f.write('')
36
36
 
37
- def setup_log_handlers(logger, log_file_name):
37
+ def setup_log_handlers(logger_name, log_file_name, log_folder_path):
38
+ check_log_dir_exists(log_folder_path)
39
+ logger = logging.getLogger(logger_name)
40
+ logger.setLevel(logging.INFO)
38
41
  check_log_file_exists(log_file_name)
42
+ if logger.hasHandlers():
43
+ logger.handlers.clear()
39
44
  file_handler = logging.FileHandler(filename=log_file_name, mode='a')
40
45
  file_handler.setFormatter(logging.Formatter('%(levelname)s - %(name)s - %(message)s'))
41
46
  logger.addHandler(file_handler)
@@ -45,57 +50,20 @@ def setup_log_handlers(logger, log_file_name):
45
50
  def get_logger(logger_name: str):
46
51
  log_folder_path = os.path.join("logs")
47
52
  file_name = os.path.join(log_folder_path, "log.txt")
48
-
49
- file_logger = logging.getLogger(logger_name)
50
- file_logger.setLevel(logging.INFO)
51
- check_log_dir_exists(log_folder_path)
52
- if file_logger.hasHandlers():
53
- return file_logger
54
-
55
- file_logger = setup_log_handlers(file_logger, file_name)
53
+
54
+ file_logger = setup_log_handlers(logger_name, file_name, log_folder_path)
56
55
 
57
56
  return file_logger
58
57
 
59
- def get_package_logger(logger_name:str):
58
+ def get_package_logger(logger_name:str, package_name:str = None):
60
59
  log_folder_path = os.path.join("logs", "package_logs")
61
- package_name = os.environ.get('PROCESSING_PACKAGE_NAME')
60
+ if package_name is None:
61
+ package_name = os.environ.get('PROCESSING_PACKAGE_NAME')
62
62
  file_name = os.path.join(log_folder_path, f"{package_name}.txt")
63
-
64
- file_logger = logging.getLogger(logger_name)
65
- file_logger.setLevel(logging.INFO)
66
- check_log_dir_exists(log_folder_path)
67
- if file_logger.hasHandlers():
68
- return file_logger
69
63
 
70
- file_logger = setup_log_handlers(file_logger, file_name)
71
-
72
- return file_logger
64
+ file_logger = setup_log_handlers(logger_name, file_name, log_folder_path)
73
65
 
74
- def log_subprocess_ouput(subprocess_out: subprocess.CompletedProcess, logger: logging.Logger):
75
- if subprocess_out.stdout:
76
- logger.info(subprocess_out.stdout)
77
- if subprocess_out.stderr:
78
- msgs = subprocess_out.stderr.split('\n')
79
- for msg in msgs:
80
- if msg is None or msg == "":
81
- continue
82
- logger.warning(msg)
83
- if subprocess_out.returncode != 0:
84
- logger.error(f"Subprocess failed with return code {subprocess_out.returncode}")
85
- raise RuntimeError()
86
-
87
- def log_git_clone_subprocess_ouput(subprocess_out: subprocess.CompletedProcess, logger: logging.Logger):
88
- if subprocess_out.stdout:
89
- logger.info(subprocess_out.stdout)
90
- if subprocess_out.stderr:
91
- msgs = subprocess_out.stderr.split('\n')
92
- for msg in msgs:
93
- if msg is None or msg == "":
94
- continue
95
- logger.info(msg)
96
- if subprocess_out.returncode != 0:
97
- logger.error(f"Subprocess failed with return code {subprocess_out.returncode}")
98
- raise RuntimeError()
66
+ return file_logger
99
67
 
100
68
  def counts_errors_warnings(log_file_path):
101
69
  error_count = 0
@@ -160,20 +128,53 @@ def output_log_by_log_level():
160
128
  log_file_path = os.path.join(package_logs_folder, log_file)
161
129
  print_out_log_by_log_level(parse_log(log_file_path), log_level)
162
130
 
163
- async def run_async_subprocess(exe_path, cmd):
164
- process = await asyncio.create_subprocess_exec(
165
- exe_path, *cmd,
166
- stdout=asyncio.subprocess.PIPE,
167
- stderr=asyncio.subprocess.PIPE
168
- )
131
+ async def run_async_subprocess(exe_path, cmd, logger, cwd=None):
132
+ if cwd is None:
133
+ process = await asyncio.create_subprocess_exec(
134
+ exe_path, *cmd,
135
+ stdout=asyncio.subprocess.PIPE,
136
+ stderr=asyncio.subprocess.PIPE
137
+ )
138
+ else:
139
+ process = await asyncio.create_subprocess_exec(
140
+ exe_path, *cmd,
141
+ cwd=cwd,
142
+ stdout=asyncio.subprocess.PIPE,
143
+ stderr=asyncio.subprocess.PIPE,
144
+ )
145
+ stdout, stderr = await process.communicate()
146
+ if process.returncode != 0:
147
+ msg = stderr.decode('utf-8')
148
+ if msg != None and msg != "":
149
+ logger.error(msg)
150
+ raise subprocess.CalledProcessError(process.returncode, cmd, stdout, stderr)
151
+ else:
152
+ msg = stdout.decode('utf-8')
153
+ if msg != None and msg != "":
154
+ logger.info(msg)
155
+
156
+ async def run_async_subprocess_without_executable(cmd, logger, cwd=None):
157
+ if cwd is None:
158
+ process = await asyncio.create_subprocess_exec(
159
+ *cmd,
160
+ stdout=asyncio.subprocess.PIPE,
161
+ stderr=asyncio.subprocess.PIPE
162
+ )
163
+ else:
164
+ process = await asyncio.create_subprocess_exec(
165
+ *cmd,
166
+ cwd=cwd,
167
+ stdout=asyncio.subprocess.PIPE,
168
+ stderr=asyncio.subprocess.PIPE,
169
+ )
170
+
169
171
  stdout, stderr = await process.communicate()
170
- py2docfx_logger = get_logger(__name__)
171
172
  if process.returncode != 0:
172
173
  msg = stderr.decode('utf-8')
173
174
  if msg != None and msg != "":
174
- py2docfx_logger.error(msg)
175
- raise RuntimeError()
175
+ logger.error(msg)
176
+ raise subprocess.CalledProcessError(process.returncode, cmd, stdout, stderr)
176
177
  else:
177
178
  msg = stdout.decode('utf-8')
178
179
  if msg != None and msg != "":
179
- py2docfx_logger.info(msg)
180
+ logger.info(msg)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: py2docfx
3
- Version: 0.1.11rc1996319
3
+ Version: 0.1.11rc1997820
4
4
  Summary: A package built based on Sphinx which download source code package and generate yaml files supported by docfx.
5
5
  Author: Microsoft Corporation
6
6
  License: MIT License
@@ -1,41 +1,41 @@
1
1
  py2docfx/__init__.py,sha256=kPRhPGPC1JknDotkksG428c1iIgfFr_4_7Jm-llrowY,72
2
- py2docfx/__main__.py,sha256=MA5jRTbG6NH5ma7Ew8iSmaTp3YSTWoRhEwOiPX2chvs,5674
2
+ py2docfx/__main__.py,sha256=DWLEx3XbEmDbGrwe1QWm5dBW_a8FJdzp_7uQeXaEgN0,5928
3
3
  py2docfx/convert_prepare/__init__.py,sha256=XxtxrP0kmW3ZBHIAoxsPDEHzcgeC0WSnole8Lk6CjKs,11
4
4
  py2docfx/convert_prepare/arg_parser.py,sha256=Wa1iK8a0Gb3QqaS4X7nUPGvh-7mWhuBS9IVbmnjC770,7606
5
5
  py2docfx/convert_prepare/constants.py,sha256=RC5DqNkqWvx4hb91FrajZ1R9dBFLxcPyoEJ43jdm36E,102
6
- py2docfx/convert_prepare/environment.py,sha256=j2pQ7cfkxJDpw9kXL4VbVuq23B3fXMoc8EdIGmLX5YM,6985
6
+ py2docfx/convert_prepare/environment.py,sha256=ClUnNHehYjxMwS50cqnCQO5vHCQaA289opSdfhkOARc,7293
7
7
  py2docfx/convert_prepare/generate_conf.py,sha256=wqs6iyElzJarH-20_qEL9zvZvt5xfBMsGXSXPSZy6wg,2295
8
- py2docfx/convert_prepare/generate_document.py,sha256=mNfJpLViIfzwRnjVwXmg5H5lVWzKJkUHeXTiJouA39o,2879
9
- py2docfx/convert_prepare/get_source.py,sha256=6EMSLiDOfid4d7SDxweV_-TGV4McxhSxKQPJju2Mi4Y,5221
10
- py2docfx/convert_prepare/git.py,sha256=OIoaX_0LG5ueY8zmUwrbxl_YyMxwLzgTp6g0uSkC2Y0,6617
11
- py2docfx/convert_prepare/install_package.py,sha256=hATmgazcSX7k2n4jQXh9sQMyNUc1k1YqHv5K5UMALq4,262
12
- py2docfx/convert_prepare/pack.py,sha256=k8eme75VRhdsZOc9ox0zNORXP4kFhAiPiP4mUdNBwwU,1614
13
- py2docfx/convert_prepare/package_info.py,sha256=BBpiuLpnkbxio_k4H6afPQPPKQEMQYO62kvJQK3BDMg,7698
8
+ py2docfx/convert_prepare/generate_document.py,sha256=iTOCMBwkfvZEj5dlEGUq4qynUodaYxJuIrG9R5eGk38,2949
9
+ py2docfx/convert_prepare/get_source.py,sha256=Sw5QMWWbKLSznHgNu7JLSW0Wy_FEk0uR_IaKKSKUCl4,5280
10
+ py2docfx/convert_prepare/git.py,sha256=Qb6uexUo3KKwp2Hq9Q27gMU4Z2P8ZBk7JcSmxBfFCBY,6491
11
+ py2docfx/convert_prepare/install_package.py,sha256=aJxQBwLRPTIKpdtLVl-SaXPaF_OX_H1ZtWmcKD8Zzuk,274
12
+ py2docfx/convert_prepare/pack.py,sha256=Jjj9ps8ESoKUFmSk6VgNmxOwMhuwirnQ-rhqigH-4VY,1578
13
+ py2docfx/convert_prepare/package_info.py,sha256=-zrMNeAkHxxzLRjBl1kRehUJy_ugc16yns-xdcAHQIQ,7711
14
14
  py2docfx/convert_prepare/package_info_extra_settings.py,sha256=u5B5e8hc0m9PA_-0kJzq1LtKn-xzZlucwXHTFy49mDg,1475
15
15
  py2docfx/convert_prepare/params.py,sha256=PXMB8pLtb4XbfI322avA47q0AO-TyBE6kZf7FU8I6v4,1771
16
16
  py2docfx/convert_prepare/paths.py,sha256=964RX81Qf__rzXgEATfqBNFCKTYVjLt9J7WCz2TnNdc,485
17
- py2docfx/convert_prepare/pip_utils.py,sha256=GPWGUBcxW8JKreGwxZIAe9AWAI78aL1aFan8DvaAf1o,2228
17
+ py2docfx/convert_prepare/pip_utils.py,sha256=Y_xmIE0Ld-gXgR8rkXSfj-AMh1bcUJvscMgSdNZyPQU,2138
18
18
  py2docfx/convert_prepare/repo_info.py,sha256=6ASJlhBwf6vZTSENgrWCVlJjlJVhuBxzdQyWEdWAC4c,117
19
19
  py2docfx/convert_prepare/source.py,sha256=6-A7oof3-WAQcQZZVpT9pKiFLH4CCIZeYqq0MN0O3gw,1710
20
- py2docfx/convert_prepare/sphinx_caller.py,sha256=4hYjDQgwEaAkCl8V7q05sn4RmWJQA_QthtztV6l6K-w,4082
20
+ py2docfx/convert_prepare/sphinx_caller.py,sha256=K7HZddkzENu_QkOrgoKeILsXkRmyQrpUXhRsow-51I4,4696
21
21
  py2docfx/convert_prepare/subpackage.py,sha256=mXAi_07pXvnPkSLZfykDh_7VeFxfLy74pYlzhMO8N_Q,5183
22
22
  py2docfx/convert_prepare/utils.py,sha256=TwfDjIqIx_nAoSeQ-LX30h_-d96WhPQcAu7P8N_tSjM,1618
23
23
  py2docfx/convert_prepare/conf_templates/conf.py_t,sha256=8zxvY1WiG-z2aiSNDY0719C08QxZLXXEMwKfYSGN0ZE,3811
24
24
  py2docfx/convert_prepare/conf_templates/root_doc.rst_t,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
25
- py2docfx/convert_prepare/post_process/merge_toc.py,sha256=DPUvL6eS0Q4gcp_fdGgWQJeYmt0sa9voWnVe0O-2giM,1265
25
+ py2docfx/convert_prepare/post_process/merge_toc.py,sha256=ENvboa3ii4WD5MH6fkHTI5PAtzAHrTrV1E1oQINSf8g,1287
26
26
  py2docfx/convert_prepare/subpackage_merge/merge_root_package.py,sha256=uK96qL2asuSfo_3SZaoP8XZaUvjf5mNkr17JNbZR4Lg,1026
27
27
  py2docfx/convert_prepare/subpackage_merge/merge_toc.py,sha256=nkVqe8R0m8D6cyTYV7aIpMDXorvn4-LXfU_vIK_hJBg,1706
28
28
  py2docfx/convert_prepare/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
29
- py2docfx/convert_prepare/tests/test_environment.py,sha256=1jONzb7saZN46h1GVI1jAvO8ws-UrjiQzl7bGEAvZBM,2131
30
- py2docfx/convert_prepare/tests/test_generate_document.py,sha256=Llt3szDunwmFKhHxc8fKuov_ci5rQokMcOgu5QNqt0U,2572
31
- py2docfx/convert_prepare/tests/test_get_source.py,sha256=TXaIPjKyA-KJopQBp8msUubT-rJLaq3llf1upfa10RM,7709
32
- py2docfx/convert_prepare/tests/test_pack.py,sha256=46JWMNzknIptDVs7D3CuxcmqBr_OKMmaw1br9H7wqco,4134
29
+ py2docfx/convert_prepare/tests/test_environment.py,sha256=UfeTlfdZKLFcUrSNODw0mvkzZSdbQHBUoK56bdm1xz8,2065
30
+ py2docfx/convert_prepare/tests/test_generate_document.py,sha256=mY8DRT-WRGIBFdDRdFwa7ZSUQwR-fTQtyKwzrzFZLU8,2621
31
+ py2docfx/convert_prepare/tests/test_get_source.py,sha256=IGBLAemm5SWUnNJJh6GJE5gBtAAdlC_cxS1xSeugzBI,7949
32
+ py2docfx/convert_prepare/tests/test_pack.py,sha256=Gw-LkIcbQuQHpoSbvkybYt4_fVw-LL12dceWw5AUKdE,4242
33
33
  py2docfx/convert_prepare/tests/test_package_info.py,sha256=3B-IzmUjETVO-5s3g3Lmh2E6JgopwnRauv8mB-SDZEM,3361
34
- py2docfx/convert_prepare/tests/test_params.py,sha256=OIsnC4MZ56xzAFBfpOF2rpg8tk3LRIfxltStG8xYrP0,2247
34
+ py2docfx/convert_prepare/tests/test_params.py,sha256=itwmVdBMtU1qIXAGaIoaDfvTOYyAL2B_WLsaBV9KUZY,2232
35
35
  py2docfx/convert_prepare/tests/test_post_process_merge_toc.py,sha256=YKOcn4_lf4syGsAvJ9BqpdUUc3SLfK4TiOX1lpXJT_Y,885
36
36
  py2docfx/convert_prepare/tests/test_source.py,sha256=LNFZtvjz6QhVLOxatjWokYCCcoSm0bhTikMF9KoTPIE,2025
37
- py2docfx/convert_prepare/tests/test_sphinx_caller.py,sha256=1ZR3uoNl1b_cpVRBcoqVpYry6tT4gNhYX_r5OpGC36M,2612
38
- py2docfx/convert_prepare/tests/test_subpackage.py,sha256=n0lCcdrTE1gkmmfGE85tSBMlpOEBszZafaHXffxo3Oc,4982
37
+ py2docfx/convert_prepare/tests/test_sphinx_caller.py,sha256=HB-roXpS1c5kr0KP69B2RzCAQ6WyxNlks0w3VTv2cIg,2786
38
+ py2docfx/convert_prepare/tests/test_subpackage.py,sha256=famt8LqwS8hCujtzvRLE5Ih8pa7t45y0d1EBk3UdYns,4984
39
39
  py2docfx/convert_prepare/tests/data/generate_document/azure-dummy-sourcecode/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
40
40
  py2docfx/convert_prepare/tests/data/generate_document/azure-dummy-sourcecode/azure/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
41
41
  py2docfx/convert_prepare/tests/data/generate_document/azure-dummy-sourcecode/azure/dummy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -57,7 +57,7 @@ py2docfx/convert_prepare/tests/data/subpackage/azure-mgmt-containerservice/azure
57
57
  py2docfx/convert_prepare/tests/data/subpackage/azure-mgmt-containerservice/azure/mgmt/containerservice/v2018_03_31/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
58
58
  py2docfx/convert_prepare/tests/data/subpackage/azure-mgmt-containerservice/azure/mgmt/containerservice/v2018_03_31/models.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
59
59
  py2docfx/docfx_yaml/__init__.py,sha256=KCEizAXv-SXtrYhvFfLHdBWDhz51AA9uagaeTL-Itpo,100
60
- py2docfx/docfx_yaml/build_finished.py,sha256=BPsb8YiJjIwiUDwVpvsHjFjXnND2zwhM--LNsDwtrsM,13823
60
+ py2docfx/docfx_yaml/build_finished.py,sha256=M0YOIgr6lQY1lqwiAIDykhnsfOtTJ1ytKxmCr0JQp_E,13826
61
61
  py2docfx/docfx_yaml/build_init.py,sha256=lAw-fnBVQbySfZ7Sut_NpFQUjnqLOmnGQrTBBH2RXcg,1860
62
62
  py2docfx/docfx_yaml/common.py,sha256=UN1MUmjUoN1QSFDR1Cm_bfRuHr6FQiOe5VQV6s8xzjc,6841
63
63
  py2docfx/docfx_yaml/convert_class.py,sha256=YaPjxqNy0p6AXi3H3T0l1MZeYRZ3dWkqusO4E0KT9QU,2161
@@ -65,7 +65,7 @@ py2docfx/docfx_yaml/convert_enum.py,sha256=HX6qdjDbdbBblx-TH41JXBLIJY3wzL3if-qnL
65
65
  py2docfx/docfx_yaml/convert_module.py,sha256=GptO1MRwaQ2Qbu724F0kCDDQQTZe7mWOtrOp3Rzgl-I,2259
66
66
  py2docfx/docfx_yaml/convert_package.py,sha256=Ep7PmvoLInDvY6OU5dveR6iVwyzGRkW3q6lX7yGJ0JE,2109
67
67
  py2docfx/docfx_yaml/directives.py,sha256=zVVuNM_6AU9G6sbqL1UAyHHgPe7bkBWbthXI-PO5ez0,879
68
- py2docfx/docfx_yaml/logger.py,sha256=XIznVK30tjpcstpyjNuJM43gs0DkCRjmU_kaLhCxS8M,6547
68
+ py2docfx/docfx_yaml/logger.py,sha256=gz416vqSqmsD91qLOCLAzomczlCfWqGXFJFRzLlVbJE,6503
69
69
  py2docfx/docfx_yaml/miss_reference.py,sha256=Btoj9wAvA4u_wU7JHH0Cei3910N8a7MS34OUqJvXAd4,2443
70
70
  py2docfx/docfx_yaml/nodes.py,sha256=tBDi35jLJArlobl07DKOkmH2qz7dudXLp_kTUfR_r2w,412
71
71
  py2docfx/docfx_yaml/parameter_utils.py,sha256=zGSIQrUfbXf9PUK-W_1K83Uo5Zk797Zlze6aMurbHIA,8706
@@ -4193,7 +4193,7 @@ py2docfx/venv/venv1/Lib/site-packages/win32comext/taskscheduler/test/test_addtas
4193
4193
  py2docfx/venv/venv1/Lib/site-packages/win32comext/taskscheduler/test/test_localsystem.py,sha256=08ojAS48W6RLsUbRD45j0SJhg_Y2NFHZT6qjT4Vrig0,75
4194
4194
  py2docfx/venv/venv1/Scripts/pywin32_postinstall.py,sha256=u95n7QQUxpCjrZistYE-3gN451zXzopuJna8cXRQ4Jw,28115
4195
4195
  py2docfx/venv/venv1/Scripts/pywin32_testall.py,sha256=-6yvZmd2lPQc4e8i6PgLsr_totF6mScvoq0Jqr0V2fM,3844
4196
- py2docfx-0.1.11rc1996319.dist-info/METADATA,sha256=xuNbUZKLC-NjBOEoJwK0I72hX6zOVJyQ35JLOngD85E,599
4197
- py2docfx-0.1.11rc1996319.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
4198
- py2docfx-0.1.11rc1996319.dist-info/top_level.txt,sha256=5dH2uP81dczt_qQJ38wiZ-gzoVWasfiJALWRSjdbnYU,9
4199
- py2docfx-0.1.11rc1996319.dist-info/RECORD,,
4196
+ py2docfx-0.1.11rc1997820.dist-info/METADATA,sha256=F20YeKnQ0qxN0lNiS6znQfX9nRpZti2_1GZkYydKsKc,599
4197
+ py2docfx-0.1.11rc1997820.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
4198
+ py2docfx-0.1.11rc1997820.dist-info/top_level.txt,sha256=5dH2uP81dczt_qQJ38wiZ-gzoVWasfiJALWRSjdbnYU,9
4199
+ py2docfx-0.1.11rc1997820.dist-info/RECORD,,