py2docfx 0.1.9.dev1897607__py3-none-any.whl → 0.1.9.dev1926139__py3-none-any.whl

py2docfx/convert_prepare/pip_utils.py

@@ -1,6 +1,8 @@
 import subprocess
 import asyncio
+import os
 from py2docfx import PACKAGE_ROOT
+from py2docfx.docfx_yaml.logger import get_logger, log_subprocess_ouput
 
 PYPI = "pypi"
 
@@ -10,10 +12,9 @@ pip_install_common_options = [
     "--no-compile",
     "--no-warn-conflicts",
     "--disable-pip-version-check",
-    "--verbose",
+    "--verbose"
 ]
 
-
 def download(package_name, path, extra_index_url=None, prefer_source_distribution=True):
     # Downloads a package from PyPI to the specified path using pip.
     download_param = ["pip", "download", "--dest", path, "--no-deps", package_name]
@@ -24,7 +25,9 @@ def download(package_name, path, extra_index_url=None, prefer_source_distributio
         download_param.append("--no-binary=:all:")
     else:
         download_param.append("--prefer-binary")
-    subprocess.run(download_param, check=True, cwd=PACKAGE_ROOT)
+    output = subprocess.run(download_param, check=True, cwd=PACKAGE_ROOT, capture_output=True, text=True)
+    docfx_logger = get_logger(__name__)
+    log_subprocess_ouput(output, docfx_logger)
 
 
 def install(package_name, options):
@@ -32,16 +35,34 @@ def install(package_name, options):
     install_param = "pip install {} {}".format(
         " ".join(pip_install_common_options + options), package_name
     ).split(" ")
-    subprocess.run(install_param, check=True, cwd=PACKAGE_ROOT)
-
+    output = subprocess.run(install_param, check=True, cwd=PACKAGE_ROOT, capture_output=True, text=True)
+    docfx_logger = get_logger(__name__)
+    log_subprocess_ouput(output, docfx_logger)
 
 def install_in_exe(exe_path, package_name, options):
     # Installs a package from PyPI using pip.
     install_param = [exe_path] + "-m pip install {} {}".format(
         " ".join(pip_install_common_options + options), package_name
     ).split(" ")
-    subprocess.run(install_param, check=True, cwd=PACKAGE_ROOT)
-
+    output = subprocess.run(install_param, check=True, cwd=PACKAGE_ROOT, capture_output=True, text=True)
+    docfx_logger = get_logger(__name__)
+    log_subprocess_ouput(output, docfx_logger)
+    
 async def install_in_exe_async(exe_path, package_name, options):
     pip_cmd = ["-m", "pip", "install"]+ pip_install_common_options + options + [package_name]
-    await(await asyncio.create_subprocess_exec(exe_path, *pip_cmd)).wait()
+    await run_async_subprocess(exe_path, pip_cmd)
+    # await(await asyncio.create_subprocess_exec(exe_path, *pip_cmd)).wait()
+    
+async def run_async_subprocess(exe_path, cmd):
+    process = await asyncio.create_subprocess_exec(
+        exe_path, *cmd,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE
+    )
+    stdout, stderr = await process.communicate()
+    docfx_logger = get_logger(__name__)
+    if process.returncode != 0:
+        docfx_logger.error(stderr.decode('utf-8'))
+        raise RuntimeError()
+    else:
+        docfx_logger.info(stdout.decode('utf-8'))

py2docfx/convert_prepare/post_process/merge_toc.py

@@ -2,10 +2,13 @@ from __future__ import annotations # Avoid A | B annotation break under <= py3.9
 import os
 import shutil
 
+from py2docfx.docfx_yaml.logger import get_logger
+
 TOC_FILE_PATH = "toc.yml"
 PACKAGE_TOC_FILE_PATH = "_build/docfx_yaml/toc.yml"
 def merge_toc(
         root_doc_path: str | os.PathLike, package_doc_path: str | os.PathLike):
+    docfx_logger = get_logger(__name__)
     root_toc_path = os.path.join(root_doc_path, TOC_FILE_PATH)
     package_toc_path = os.path.join(package_doc_path, PACKAGE_TOC_FILE_PATH)
 
@@ -18,7 +21,8 @@ def merge_toc(
             if not toc_content.endswith("\n"):
                 root_toc_handle.write("\n")
     else:
-        raise ValueError(f"TOC content empty: {package_toc_path}")
+        docfx_logger.error(f"TOC content empty: {package_toc_path}")
+        raise ValueError()
 
     # delete package toc.yml
     os.remove(package_toc_path)

py2docfx/convert_prepare/sphinx_caller.py

@@ -5,6 +5,7 @@ import sphinx.ext.apidoc as apidoc
 import subprocess
 
 from py2docfx import PACKAGE_ROOT
+from py2docfx.docfx_yaml.logger import get_logger, log_subprocess_ouput
 from py2docfx.convert_prepare.package_info import PackageInfo
 from py2docfx.convert_prepare.paths import folder_is_hidden
 from py2docfx.convert_prepare.subpackage import (get_subpackages,
@@ -19,6 +20,7 @@ def run_apidoc(rst_path, source_code_path, exclude_paths, package_info: PackageI
     Replacing
     https://apidrop.visualstudio.com/Content%20CI/_git/ReferenceAutomation?path=/Python/build.ps1&line=110&lineEnd=126&lineStartColumn=1&lineEndColumn=14&lineStyle=plain&_a=contents
     """
+    docfx_logger = get_logger(__name__)
     subfolderList = [name for name in
                        os.listdir(source_code_path)
                        if os.path.isdir(os.path.join(source_code_path, name))
@@ -28,16 +30,29 @@ def run_apidoc(rst_path, source_code_path, exclude_paths, package_info: PackageI
     for subfolder in subfolderList:
         subfolderPath = os.path.join(source_code_path, subfolder)
         if os.path.isdir(subfolderPath):
-            print("<CI INFO>: Subfolder path {}.".format(subfolderPath))
-            args = [
-                "--module-first",
-                "--no-headings",
-                "--no-toc",
-                "--implicit-namespaces",
-                "-o",
-                rst_path,
-                subfolderPath,
-            ]
+            msg = "<CI INFO>: Subfolder path {}.".format(subfolderPath)
+            docfx_logger.info(msg)
+            if os.environ.get('LOG_LEVEL') == 'INFO':
+                args = [
+                    "--module-first",
+                    "--no-headings",
+                    "--no-toc",
+                    "--implicit-namespaces",
+                    "-o",
+                    rst_path,
+                    subfolderPath,
+                ]
+            else:
+                args = [
+                    "-q",
+                    "--module-first",
+                    "--no-headings",
+                    "--no-toc",
+                    "--implicit-namespaces",
+                    "-o",
+                    rst_path,
+                    subfolderPath
+                ]
             args.extend(exclude_paths)
             apidoc.main(args)
             if package_info.build_in_subpackage and subfolder == "azure":
@@ -56,14 +71,16 @@ def run_converter(rst_path, out_path, sphinx_build_path: str, extra_package_path
     Replacing
     https://apidrop.visualstudio.com/Content%20CI/_git/ReferenceAutomation?path=/Python/build.ps1&line=150&lineEnd=161&lineStartColumn=13&lineEndColumn=52&lineStyle=plain&_a=contents
     """
-
+    docfx_logger = get_logger(__name__)
     outdir = os.path.join(out_path, "_build")
 
     # Sphinx/docutils have memory leak including linecaches, module-import-caches,
     # Use a subprocess on production to prevent out of memory
 
     if not sys.executable:
-        raise ValueError("Can't get the executable binary for the Python interpreter.")
+        msg = "Can't get the executable binary for the Python interpreter."
+        docfx_logger.error(msg)
+        raise ValueError()
     sphinx_param = [
         executable,
         sphinx_build_path,
@@ -82,6 +99,6 @@ def run_converter(rst_path, out_path, sphinx_build_path: str, extra_package_path
         env_tmp["PYTHONPATH"] = f"{extra_package_path};{package_root_parent};"
     else:
         env_tmp["PYTHONPATH"] = f"{extra_package_path}:{package_root_parent}:"
-    subprocess.run(sphinx_param, check=True, cwd=PACKAGE_ROOT, env=env_tmp)
-
+    output = subprocess.run(sphinx_param, check=True, cwd=PACKAGE_ROOT, env=env_tmp, capture_output=True ,text=True)
+    log_subprocess_ouput(output, docfx_logger)
     return outdir

py2docfx/convert_prepare/tests/test_generate_document.py

@@ -10,15 +10,17 @@ import sphinx.cmd.build
 from py2docfx.convert_prepare.generate_document import generate_document
 from py2docfx.convert_prepare.package_info import PackageInfo
 from py2docfx.convert_prepare.source import Source
+from py2docfx.convert_prepare.tests.utils import prepare_log_folder
 
 def test_generate_document(tmp_path):
     """
     Test the generate_document function.
     """
     # init test case
-    source_code_path = "convert_prepare\\tests\\data\\generate_document"
+    prepare_log_folder()
+    source_code_path = os.path.join("convert_prepare", "tests", "data", "generate_document")
     output_root = os.path.join(tmp_path, "output")
-    shutil.copytree(source_code_path, os.path.join(tmp_path, "source\\0"))
+    shutil.copytree(source_code_path, os.path.join(tmp_path, "source", "0"))
     package = PackageInfo()
     package.name = "azure-dummy-sourcecode"
     package.exclude_path = ["test*", "example*", "sample*", "doc*"]
@@ -27,7 +29,7 @@ def test_generate_document(tmp_path):
     package.build_in_subpackage = False
     package.extra_index_url = None
     package.prefer_source_distribution = False
-    source_folder = os.path.join(tmp_path, "source\\0\\azure-dummy-sourcecode")
+    source_folder = os.path.join(tmp_path, "source", "0", "azure-dummy-sourcecode")
     yaml_output_folder = os.path.join(tmp_path, "yaml_output")
     package.path = Source(
         source_folder=source_folder, yaml_output_folder=yaml_output_folder, package_name=package.name
@@ -45,7 +47,7 @@ def test_generate_document(tmp_path):
     generate_document(package, output_root, sphinx_build_path = sphinx.cmd.build.__file__, extra_package_path = source_folder)
 
     #assert the result
-    yaml_path = os.path.join(yaml_output_folder, "_build\\docfx_yaml")
+    yaml_path = os.path.join(yaml_output_folder, "_build", "docfx_yaml")
     assert os.path.exists(yaml_path)
     assert os.path.exists(os.path.join(yaml_path, "azure.dummy.sourcecode.foo.foo.yml"))
     assert os.path.exists(os.path.join(yaml_path, "azure.dummy.sourcecode.boo.boo.yml"))

py2docfx/convert_prepare/tests/test_get_source.py

@@ -94,7 +94,7 @@ def test_get_source_dist_file_zip(init_package_info):
     package.location = "https://files.pythonhosted.org/packages/3e/71/f6f71a276e2e69264a97ad39ef850dca0a04fce67b12570730cb38d0ccac/azure-common-1.1.28.zip"
     get_source.get_source(sys.executable, package, 1)
     assert os.path.exists("dist_temp/1/azure-common-1.1.28")
-    assert package.path.source_folder == "dist_temp\\1\\azure-common-1.1.28"
+    assert package.path.source_folder == os.path.join("dist_temp", "1", "azure-common-1.1.28")
     assert os.path.exists("dist_temp/1/azure-common-1.1.28.zip") is False
 
 def test_get_source_dist_file_whl(init_package_info):
@@ -112,7 +112,7 @@ def test_get_source_dist_file_whl(init_package_info):
     package.location = "https://files.pythonhosted.org/packages/62/55/7f118b9c1b23ec15ca05d15a578d8207aa1706bc6f7c87218efffbbf875d/azure_common-1.1.28-py2.py3-none-any.whl"
     get_source.get_source(sys.executable, package, 2)
     assert os.path.exists("dist_temp/2/azure_common-1.1.28")
-    assert package.path.source_folder == "dist_temp\\2\\azure_common-1.1.28"
+    assert package.path.source_folder == os.path.join("dist_temp", "2", "azure_common-1.1.28")
     assert os.path.exists("dist_temp/2/azure_common-1.1.28-py2.py3-none-any.whl") is False
 
 def test_get_source_dist_file_tar(init_package_info):
@@ -130,7 +130,7 @@ def test_get_source_dist_file_tar(init_package_info):
     package.location = "https://files.pythonhosted.org/packages/fa/19/43a9eb812b4d6071fdc2c55640318f7eb5a1be8dbd3b6f9d96a1996e1bb6/azure-core-1.29.4.tar.gz"
     get_source.get_source(sys.executable, package, 3)
     assert os.path.exists("dist_temp/3/azure-core-1.29.4")
-    assert package.path.source_folder == "dist_temp\\3\\azure-core-1.29.4"
+    assert package.path.source_folder == os.path.join("dist_temp", "3", "azure-core-1.29.4")
     assert os.path.exists("dist_temp/3/azure-core-1.29.4.tar.gz") is False
 
 def test_get_source_pip_whl(init_package_info):
@@ -147,7 +147,7 @@ def test_get_source_pip_whl(init_package_info):
     package.prefer_source_distribution = False
     get_source.get_source(sys.executable, package, 4)
     assert os.path.exists("dist_temp/4/azure_common-1.1.28")
-    assert package.path.source_folder == "dist_temp\\4\\azure_common-1.1.28"
+    assert package.path.source_folder == os.path.join("dist_temp", "4", "azure_common-1.1.28")
     assert os.path.exists("dist_temp/4/azure_common-1.1.28-py2.py3-none-any.whl") is False
 
 def test_get_source_pip_zip(init_package_info):
@@ -164,7 +164,7 @@ def test_get_source_pip_zip(init_package_info):
     package.prefer_source_distribution = True
     get_source.get_source(sys.executable, package, 5)
     assert os.path.exists("dist_temp/5/azure-common-1.1.28")
-    assert package.path.source_folder == "dist_temp\\5\\azure-common-1.1.28"
+    assert package.path.source_folder == os.path.join("dist_temp", "5", "azure-common-1.1.28")
     assert os.path.exists("dist_temp/5/azure-common-1.1.28.zip") is False
 
 def test_get_source_zip_file_at_position_0(init_package_info):
@@ -181,5 +181,5 @@ def test_get_source_zip_file_at_position_0(init_package_info):
     package.prefer_source_distribution = True
     get_source.get_source(sys.executable, package, 6)
     assert os.path.exists("dist_temp/6/azure_template-0.1.0b3942895")
-    assert package.path.source_folder == "dist_temp\\6\\azure_template-0.1.0b3942895"
+    assert package.path.source_folder == os.path.join("dist_temp", "6", "azure_template-0.1.0b3942895")
     assert os.path.exists("dist_temp/6/azure_template-0.1.0b3942895.tar.gz") is False

py2docfx/convert_prepare/tests/test_params.py

@@ -48,7 +48,7 @@ def test_load_command_params():
 def test_load_file_params():
     # get absolute path of test json
     path = os.path.dirname(os.path.abspath(__file__))
-    relative_path = "data\\params\\test.json"
+    relative_path = os.path.join("data", "params", "test.json")
     full_path = os.path.join(path, relative_path)
     json_info = load_file_params(full_path)
 

py2docfx/convert_prepare/tests/test_sphinx_caller.py

@@ -7,18 +7,19 @@ import sphinx.cmd.build
 from py2docfx.convert_prepare.sphinx_caller import run_apidoc, run_converter
 from py2docfx.convert_prepare.package_info import PackageInfo
 from py2docfx.convert_prepare.source import Source
+from py2docfx.convert_prepare.tests.utils import prepare_log_folder
 
 package_info = PackageInfo()
 
 def init_paths(tmp_path):
     rst_path = os.path.join(tmp_path, "test_rst")
 
-    source_code_path = os.path.abspath("convert_prepare\\tests\\data\\sphinx_caller\\test_source_code")
+    source_code_path = os.path.abspath("convert_prepare/tests/data/sphinx_caller/test_source_code")
     destination = os.path.join(tmp_path, "test_source_code")
     shutil.copytree(source_code_path, destination)
 
     package_info.code_location = destination
-    package_info.exclude_path = ["testcode\\exclude\\*"]
+    package_info.exclude_path = ["testcode/exclude/*"]
     package_info.path = Source(source_folder=destination, yaml_output_folder=rst_path, package_name="testcode")
     package_info.build_in_subpackage = False
     package_info.name = 'testcode'
@@ -36,11 +37,12 @@ def test_run_apidoc(tmp_path):
 
 
 def test_run_converter(tmp_path):
+    prepare_log_folder()
     rst_path, source_code_path = init_paths(tmp_path)
     run_apidoc(rst_path, source_code_path, package_info.get_exluded_command(), package_info)
 
     # prepare conf.py, index.rst and docfx_yaml
-    conf_path = os.path.abspath("convert_prepare\\tests\\data\\sphinx_caller\\conf.py")
+    conf_path = os.path.abspath("convert_prepare/tests/data/sphinx_caller/conf.py")
     destination = os.path.join(rst_path)
     shutil.copy(conf_path, destination)
     docfx_yaml_path = os.path.abspath("docfx_yaml")

py2docfx/convert_prepare/tests/utils.py

@@ -0,0 +1,11 @@
+import os
+import shutil
+
+def prepare_log_folder() -> None:
+    if not os.path.exists('logs'):
+        os.makedirs('logs')
+        os.makedirs(os.path.join('logs', 'package_logs'))
+    else:
+        shutil.rmtree('logs')
+        os.makedirs('logs')
+        os.makedirs(os.path.join('logs', 'package_logs'))

py2docfx/docfx_yaml/build_finished.py

@@ -18,6 +18,7 @@ from convert_class import convert_class
 from convert_enum import convert_enum
 from convert_module import convert_module
 from convert_package import convert_package
+from logger import get_package_logger
 
 INITPY = '__init__.py'
 MODULE = 'module'
@@ -91,6 +92,7 @@ def add_isrequired_if_needed(obj, key: str):
                 args['isRequired'] = True
 
 def get_merged_params(obj, info_field_data, key: str):
+    docfx_logger = get_package_logger(__name__)
     merged_params = []
     arg_params = obj.get('syntax', {}).get(key, [])
     if key in info_field_data[obj['uid']]:
@@ -98,7 +100,8 @@ def get_merged_params(obj, info_field_data, key: str):
             key, [])
         if arg_params and doc_params:
             if len(arg_params) - len(doc_params) > 0:
-                print(f'Documented params don\'t match size of params:{obj["uid"]}') # CodeQL: [py/clear-text-logging-sensitive-data] There is no sensitive data in the print statement.
+                msg = f'Documented params don\'t match size of params:{obj["uid"]}' # CodeQL: [py/clear-text-logging-sensitive-data] There is no sensitive data in the print statement.
+                docfx_logger.warning(msg)
             doc_params = remove_params_without_id(doc_params)
             merged_params = merge_params(arg_params, doc_params)
     else:
@@ -182,6 +185,8 @@ def build_finished(app, exception):
     """
     Output YAML on the file system.
     """
+    
+    docfx_logger = get_package_logger(__name__)
 
     def convert_class_to_enum_if_needed(obj):
         if (obj.get('inheritance'), None):
@@ -299,7 +304,8 @@ def build_finished(app, exception):
                 mime = "PythonModule"
 
             if transformed_obj == None:
-                print(f"Unknown yml, uid is: {uid}")
+                msg = f"Unknown yml, uid is: {uid}"
+                docfx_logger.warning(msg)
             else:
                 # save file
                 common.write_yaml(transformed_obj, out_file, mime)
@@ -307,7 +313,9 @@ def build_finished(app, exception):
 
     # Write TOC, the toc should include at least 1 
     if len(toc_yaml) == 0:
-        raise RuntimeError("No documentation for this module.")
+        msg = "No documentation for this module."
+        docfx_logger.error(msg)
+        raise RuntimeError()
 
     toc_file = os.path.join(normalized_outdir, 'toc.yml')
     with open(toc_file, 'w') as writable:

py2docfx/docfx_yaml/convert_class.py

@@ -3,9 +3,10 @@ import os
 import yaml as yml
 from functools import partial
 from common import remove_empty_values, parse_references, convert_member, convert_parameter, convert_types, get_constructor_and_variables, convert_variable
-
+from logger import get_package_logger
 
 def convert_class(obj):
+    docfx_logger = get_package_logger(__name__)
     record = {}
     reference_mapping = {}
     old_class_object = {}
@@ -48,5 +49,6 @@ def convert_class(obj):
     new_class_object['methods'] = list(map(convert_member_partial, methods))
     new_class_object['attributes'] = list(map(convert_member_partial, attributes))
 
-    print("class: " + new_class_object['uid']) # CodeQL: [py/clear-text-logging-sensitive-data] There is no sensitive data in the print statement.
+    msg = "class: " + new_class_object['uid'] # CodeQL: [py/clear-text-logging-sensitive-data] There is no sensitive data in the print statement.
+    docfx_logger.info(msg)
     return remove_empty_values(new_class_object)

py2docfx/docfx_yaml/convert_enum.py

@@ -1,12 +1,13 @@
 # -*- coding: utf-8 -*-
 from common import remove_empty_values, parse_references, get_constructor_and_variables, convert_variable
-
+from logger import get_package_logger
 
 def convert_enum(obj):
     ''' Convert the Enum yaml object to a SDP style object.
 
     :param obj: The object is generated from the Enum yaml file
     '''
+    docfx_logger = get_package_logger(__name__)
     record = {}
     reference_mapping = {}
     old_enum_object = {}
@@ -43,7 +44,8 @@ def convert_enum(obj):
 
     new_enum_object['fields'] = list(map(convert_fields, fields))
 
-    print("enum: " + new_enum_object['uid']) # CodeQL: [py/clear-text-logging-sensitive-data] There is no sensitive data in the print statement.
+    msg = "enum: " + new_enum_object['uid'] # CodeQL: [py/clear-text-logging-sensitive-data] There is no sensitive data in the print statement.
+    docfx_logger.info(msg)
     return remove_empty_values(new_enum_object)
 
 

py2docfx/docfx_yaml/convert_module.py

@@ -3,9 +3,10 @@ import os
 import yaml as yml
 from functools import partial
 from common import remove_empty_values, parse_references, convert_member
-
+from logger import get_package_logger
 
 def convert_module(obj, uid_type_mapping):
+    docfx_logger = get_package_logger(__name__)
     record = {}
     reference_mapping = {}
     old_object = {}
@@ -54,5 +55,6 @@ def convert_module(obj, uid_type_mapping):
         filter(lambda x: uid_type_mapping.get(x, None) == 'enum', children))
 
     toreturn = remove_empty_values(new_object)
-    print("module: " + toreturn['uid']) # CodeQL: [py/clear-text-logging-sensitive-data] There is no sensitive data in the print statement.
+    msg = "module: " + toreturn['uid'] # CodeQL: [py/clear-text-logging-sensitive-data] There is no sensitive data in the print statement.
+    docfx_logger.info(msg)
     return toreturn

py2docfx/docfx_yaml/convert_package.py

@@ -3,9 +3,10 @@ import os
 import yaml as yml
 from functools import partial
 from common import remove_empty_values, parse_references, convert_member
-
+from logger import get_package_logger
 
 def convert_package(obj, uid_type_mapping):
+    docfx_logger = get_package_logger(__name__)
     record = {}
     reference_mapping = {}
     old_object = {}
@@ -49,5 +50,6 @@ def convert_package(obj, uid_type_mapping):
         filter(lambda x: uid_type_mapping.get(x, None) == 'enum', children))
 
     toreturn = remove_empty_values(new_object)
-    print("package: " + toreturn['uid']) # CodeQL: [py/clear-text-logging-sensitive-data] There is no sensitive data in the print statement.
+    msg = "package: " + toreturn['uid'] # CodeQL: [py/clear-text-logging-sensitive-data] There is no sensitive data in the print statement.
+    docfx_logger.info(msg)
     return toreturn

py2docfx/docfx_yaml/logger.py

@@ -0,0 +1,68 @@
+import os
+import logging
+import subprocess
+
+def get_log_level():
+    if os.environ.get('LOG_LEVEL') == 'WARNING':
+        return logging.WARNING
+    elif os.environ.get('LOG_LEVEL') == 'INFO':
+        return logging.INFO
+    
+    return logging.ERROR
+
+def setup_log_handlers(logger, log_file_name):
+    file_handler = logging.FileHandler(filename=log_file_name, mode='a')
+    file_handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
+    
+    stream_handeler = logging.StreamHandler()
+    stream_handeler.setLevel(get_log_level())
+    
+    logger.addHandler(file_handler)
+    logger.addHandler(stream_handeler)
+    
+    return logger
+
+def get_logger(logger_name: str):
+    file_name = os.path.join("logs", "log.txt")
+    
+    file_logger = logging.getLogger(logger_name)
+    file_logger.setLevel(logging.INFO)
+    
+    if file_logger.hasHandlers():
+        return file_logger
+    
+    file_logger = setup_log_handlers(file_logger, file_name)
+
+    return file_logger
+
+def get_package_logger(logger_name:str):
+    package_name = os.environ.get('PROCESSING_PACKAGE_NAME')
+    file_name = os.path.join("logs", "package_logs", f"{package_name}.txt")
+    
+    file_logger = logging.getLogger(logger_name)
+    file_logger.setLevel(logging.INFO)
+
+    if file_logger.hasHandlers():
+        return file_logger
+
+    file_logger = setup_log_handlers(file_logger, file_name)
+    
+    return file_logger
+
+def log_subprocess_ouput(subprocess_out: subprocess.CompletedProcess, logger: logging.Logger):
+    if subprocess_out.stdout:
+        logger.info(subprocess_out.stdout)
+    if subprocess_out.stderr:
+        msgs = subprocess_out.stderr.split('\n')
+        for msg in msgs:
+            if msg is None or msg == "":
+                continue
+            if "WARNING:" in msg:
+                logger.warning(msg)
+            elif "ERROR:" in msg:
+                logger.error(msg)
+            else:
+                logger.info(msg)
+    if subprocess_out.returncode != 0:
+        logger.error(f"Subprocess failed with return code {subprocess_out.returncode}")
+        raise RuntimeError()

py2docfx/docfx_yaml/process_doctree.py

@@ -11,6 +11,7 @@ import re
 from utils import transform_string
 from enum import EnumMeta
 from importlib import import_module
+from logger import get_package_logger
 
 PACKAGE = 'package'
 METHOD = 'method'
@@ -181,7 +182,7 @@ def _create_datam(app, cls, module, name, _type, obj, lines=None):
     """
     Build the data structure for an autodoc class
     """
-
+    docfx_logger = get_package_logger(__name__)
     if lines is None:
         lines = []
     short_name = name.split('.')[-1]
@@ -204,7 +205,8 @@ def _create_datam(app, cls, module, name, _type, obj, lines=None):
                     args = removePositonalOnlyFromArgs(args, positional_only_params)
 
     except Exception as e:
-        print("Can't get argspec for {}: {}. Exception: {}".format(type(obj), name, e))
+        msg = "Can't get argspec for {}: {}. Exception: {}".format(type(obj), name, e)
+        docfx_logger.warning(msg)
 
     datam = {
         'module': module if module else None,
@@ -388,7 +390,7 @@ def process_docstring(app, _type, name, obj, options, lines):
     This function takes the docstring and indexes it into memory.
     """
     # Use exception as class
-
+    docfx_logger = get_package_logger(__name__)
     def check_convert_package_type(obj, _type):
         if _type == MODULE:
             filename = getattr(obj, '__file__', None)
@@ -409,7 +411,7 @@ def process_docstring(app, _type, name, obj, options, lines):
     cls, module = _get_cls_module(_type, name)
 
     if _type != PACKAGE and not module:
-        print('Unknown Type: %s' % _type)
+        docfx_logger.warning('Unknown Type: %s' % _type)
         return None
 
     if app.config.__contains__('autoclass_content') and app.config.autoclass_content.lower() == 'both':

py2docfx/docfx_yaml/translator.py

@@ -11,10 +11,10 @@ import return_type_utils
 from docutils import nodes
 from sphinx import addnodes
 from sphinx.addnodes import desc_signature,desc_content
-from sphinx.util import logging
 from sphinx.util.docfields import _is_single_paragraph
 from collections import OrderedDict
 from nodes import remarks
+from logger import get_package_logger
 
 TYPE_SEP_PATTERN = '(\[|\]|, |\(|\))'
 PARAMETER_NAME = "[*][*](.*?)[*][*]"
@@ -26,10 +26,9 @@ ATTRIBUTE_TYPE = 'attribute'
 types_contain_constructor = {CLASS_TYPE, EXCEPTION_TYPE}
 types_contain_attributes = {CLASS_TYPE, EXCEPTION_TYPE}
 
-logger = logging.getLogger(__name__)
-
 def translator(app, docname, doctree):
 
+    docfx_logger = get_package_logger(__name__)
     transform_node = app.docfx_transform_node
 
     class_obj_cache = app.env.domains['py'].objects
@@ -58,16 +57,15 @@ def translator(app, docname, doctree):
     def _get_uid_and_type_from_desc(node):
         assert node.tagname == 'desc'
         if node.attributes['domain'] != 'py':
-            print(
-                'Skipping Domain Object (%s)' % node.attributes['domain']
-            )
+            msg = str('Skipping Domain Object (%s)' % node.attributes['domain'])
+            docfx_logger.info(msg)
             return None, None
 
         try:
             module = node[0].attributes['module']
             full_name = node[0].attributes['fullname']
         except KeyError as e:
-            logger.error("There maybe some syntax error in docstring near: " + node.astext())
+            docfx_logger.error("There maybe some syntax error in docstring near: " + node.astext())
             raise e
 
         uid = '{module}.{full_name}'.format(module=module, full_name=full_name)