py2docfx 0.1.0.dev1464944__py3-none-any.whl → 0.1.0.dev1465062__py3-none-any.whl

py2docfx/convert_prepare/package_info.py

@@ -154,4 +154,7 @@ class PackageInfo:
                         for path in self.exclude_path
                     ]
                 )
+            # exclude root package __init__.py, prevent generating azure.yml
+            root_package_name = [seg for seg in self.name.split('-') if seg][0]
+            exclude_path.append(os.path.join(code_location, f'{root_package_name}/__init__.py'))
         return exclude_path

py2docfx/convert_prepare/sphinx_caller.py

@@ -1,11 +1,17 @@
 import os
 from sphinx.application import Sphinx
 import sphinx.ext.apidoc as apidoc
+import sphinx.cmd.build
+import subprocess
+from pathlib import Path
+from py2docfx import PACKAGE_ROOT
 from py2docfx.convert_prepare.package_info import PackageInfo
 from py2docfx.convert_prepare.paths import folder_is_hidden
 from py2docfx.convert_prepare.subpackage import (get_subpackages,
                                         move_rst_files_to_subfolder)
 
+DEBUG_SPHINX_FLAG = 'PY2DOCFX_DEBUG_SPHINX'
+
 def run_apidoc(rst_path, source_code_path, exclude_paths, package_info: PackageInfo):
     """
     Run sphinx-apidoc to generate RST inside rst_path folder
@@ -52,12 +58,39 @@ def run_converter(rst_path, out_path, conf_path = None):
     """
 
     outdir = os.path.join(out_path, "_build")
-    app = Sphinx(
-        srcdir=rst_path,
-        outdir=outdir,
-        confdir=conf_path or rst_path,
-        doctreedir=os.path.join(outdir, "doctrees"),
-        buildername="yaml",
-    )
-    app.build(force_all=True)
+
+    # Sphinx/docutils have memory leak including linecaches, module-import-caches,
+    # Use a subprocess on production to prevent out of memory
+    debug_into_sphinx = os.environ.get(DEBUG_SPHINX_FLAG, 'false')
+    debug_into_sphinx = debug_into_sphinx.lower() in {'true', '1'}
+    if debug_into_sphinx:
+        app = Sphinx(
+            srcdir=rst_path,
+            outdir=outdir,
+            confdir=conf_path or rst_path,
+            doctreedir=os.path.join(outdir, "doctrees"),
+            buildername="yaml",
+        )
+        app.build(force_all=True)
+    else:
+        sphinx_build_path = sphinx.cmd.build.__file__
+        sphinx_param = [
+            'python', sphinx_build_path,
+            rst_path,
+            outdir,
+            '-c', conf_path or rst_path,
+            '-d', os.path.join(outdir, "doctrees"),
+            '-b', 'yaml'
+        ]
+
+        # TODO: update generate_conf to replace "yaml_builder" with "py2docfx.docfx_yaml.yaml_builder"
+        # then no need to manually add docfx_yaml to path
+        package_root_parent = os.path.join(PACKAGE_ROOT, 'docfx_yaml')
+        env_tmp = os.environ.copy()
+        python_path_current = env_tmp.get('PYTHONPATH', None)
+        if os.name == 'nt' :
+            env_tmp["PYTHONPATH"] = f"{package_root_parent};{python_path_current}" if python_path_current else package_root_parent
+        else:
+            env_tmp["PYTHONPATH"] = f"{package_root_parent}:{python_path_current}" if python_path_current else package_root_parent
+        subprocess.run(sphinx_param, check=True, cwd=PACKAGE_ROOT, env=env_tmp)
     return outdir

py2docfx/convert_prepare/tests/test_generate_document.py

@@ -13,26 +13,26 @@ def test_generate_document(tmp_path):
     Test the generate_document function.
     """
     # init test case
-    source_code_path = "convert_prepare\\tests\\data\\generate_document\\"
+    source_code_path = "convert_prepare\\tests\\data\\generate_document"
     output_root = os.path.join(tmp_path, "output")
     shutil.copytree(source_code_path, os.path.join(tmp_path, "source\\0"))
     package = PackageInfo()
-    package.name = "dummy_source_code"
+    package.name = "dummy-sourcecode"
     package.exclude_path = ["test*", "example*", "sample*", "doc*"]
     package.install_type = PackageInfo.InstallType.PYPI
     package.version = None
     package.build_in_subpackage = False
     package.extra_index_url = None
     package.prefer_source_distribution = False
-    source_folder = os.path.join(tmp_path, "source\\0")
+    source_folder = os.path.join(tmp_path, "source\\0\\dummy-sourcecode")
     yaml_output_folder = os.path.join(tmp_path, "yaml_output")
     package.path = Source(
         source_folder=source_folder, yaml_output_folder=yaml_output_folder, package_name=package.name
     )
 
     # add dummy code to python sys path to simulate installation
-    dummy_path = os.path.join(tmp_path, "source\\0")
-    sys.path.insert(0, os.path.abspath(dummy_path))
+    os.environ["PYTHONPATH"] = os.path.abspath(source_folder)
+    sys.path.insert(0, os.path.abspath(source_folder))
 
     # add docfx_yaml to python sys path for sphinx build to import
     sys.path.insert(1, os.path.abspath("docfx_yaml"))
@@ -43,10 +43,12 @@ def test_generate_document(tmp_path):
     #assert the result
     yaml_path = os.path.join(yaml_output_folder, "_build\\docfx_yaml")
     assert os.path.exists(yaml_path)
-    assert os.path.exists(os.path.join(yaml_path, "dummy_source_code.foo.foo.yml"))
-    assert os.path.exists(os.path.join(yaml_path, "dummy_source_code.boo.boo.yml"))
-    assert os.path.exists(os.path.join(yaml_path, "dummy_source_code.foo.foo.Foo.yml"))
-    assert os.path.exists(os.path.join(yaml_path, "dummy_source_code.boo.boo.Boo.yml"))
+    assert os.path.exists(os.path.join(yaml_path, "dummy.sourcecode.foo.foo.yml"))
+    assert os.path.exists(os.path.join(yaml_path, "dummy.sourcecode.boo.boo.yml"))
+    assert os.path.exists(os.path.join(yaml_path, "dummy.sourcecode.foo.foo.Foo.yml"))
+    assert os.path.exists(os.path.join(yaml_path, "dummy.sourcecode.boo.boo.Boo.yml"))
+    assert os.path.exists(os.path.join(yaml_path, "dummy.sourcecode.yml"))
+    assert not os.path.exists(os.path.join(yaml_path, "dummy.yml"))
     assert os.path.exists(os.path.join(yaml_path, "toc.yml"))
     assert os.path.exists(os.path.join(yaml_path, "index.yml"))
 

py2docfx/convert_prepare/tests/test_package_info.py

@@ -49,6 +49,7 @@ def test_get_exclude_command(tmp_path):
         "example*",
         "sample*",
         "doc*",
+        "mltable/__init__.py"
     ]
     def form_exclude_path(raletive_path):
         return os.path.join(source_folder, raletive_path)

py2docfx/convert_prepare/tests/test_sphinx_caller.py

@@ -18,7 +18,7 @@ def init_paths(tmp_path):
     package_info.exclude_path = ["testcode\\exclude\\*"]
     package_info.path = Source(source_folder=destination, yaml_output_folder=rst_path, package_name="testcode")
     package_info.build_in_subpackage = False
-
+    package_info.name = 'testcode'
     return rst_path, destination
 
 
@@ -28,7 +28,6 @@ def test_run_apidoc(tmp_path):
 
     # List all files under rst_path
     rst_list = os.listdir(rst_path)
-    assert "testcode.rst" in rst_list
     assert "testcode.fakemodule.rst" in rst_list
     assert "testcode.exclude.rst" not in rst_list
 
@@ -51,7 +50,7 @@ def test_run_converter(tmp_path):
 
     if os.path.exists(out_path):
         yaml_list = os.listdir(os.path.join(out_path, "docfx_yaml"))
-        assert "testcode.yml" in yaml_list
+        assert "testcode.yml" not in yaml_list
         assert "testcode.fakemodule.test_code.yml" in yaml_list
         assert "testcode.fakemodule.test_code.testClass.yml" in yaml_list
         assert "toc.yml" in yaml_list

{py2docfx-0.1.0.dev1464944.dist-info → py2docfx-0.1.0.dev1465062.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: py2docfx
-Version: 0.1.0.dev1464944
+Version: 0.1.0.dev1465062
 Summary: A package built based on Sphinx which download source code package and generate yaml files supported by docfx.
 Author: Microsoft Corporation
 License: MIT License

{py2docfx-0.1.0.dev1464944.dist-info → py2docfx-0.1.0.dev1465062.dist-info}/RECORD

@@ -8,13 +8,13 @@ py2docfx/convert_prepare/get_source.py,sha256=1XOdFABdzhmSld8-w72gaOQv8_xaUX7_Vn
 py2docfx/convert_prepare/git.py,sha256=fY9urQQp5aMnlGnFrSvTTR0FPnIc-28b50W1NzIPpaw,5640
 py2docfx/convert_prepare/install_package.py,sha256=hATmgazcSX7k2n4jQXh9sQMyNUc1k1YqHv5K5UMALq4,262
 py2docfx/convert_prepare/pack.py,sha256=vZS67_GzEhUmZWHU1dxm8gnWyRBs-kB6-KjX1d_FdOU,1260
-py2docfx/convert_prepare/package_info.py,sha256=8H-v-6-ta1mx3fY2o20ra1uO1lfu2mheikbarDSpK7U,6292
+py2docfx/convert_prepare/package_info.py,sha256=ZSYDTKVqDYV6DLx6N6sD05qNbreIbjyo5IOLEhsGUrw,6549
 py2docfx/convert_prepare/params.py,sha256=PXMB8pLtb4XbfI322avA47q0AO-TyBE6kZf7FU8I6v4,1771
 py2docfx/convert_prepare/paths.py,sha256=964RX81Qf__rzXgEATfqBNFCKTYVjLt9J7WCz2TnNdc,485
 py2docfx/convert_prepare/pip_utils.py,sha256=W8PJQQSZXUW7W_mdBxaK6KRuxMEskO1-Hw6hjRazqTY,1127
 py2docfx/convert_prepare/repo_info.py,sha256=6ASJlhBwf6vZTSENgrWCVlJjlJVhuBxzdQyWEdWAC4c,117
 py2docfx/convert_prepare/source.py,sha256=reRr3zPG_Q8zCGaJf-7JbaO0A58RCQAURiGYamx6LEM,1449
-py2docfx/convert_prepare/sphinx_caller.py,sha256=Y5VdYfAkqhcr6RHEOL4I5KiBkrX1APJKplTPcX6m34Y,2599
+py2docfx/convert_prepare/sphinx_caller.py,sha256=ZeCEtJPdondwX50Gso6nPwre5VUqtVoBTtbWJ5ID5tU,4132
 py2docfx/convert_prepare/subpackage.py,sha256=mXAi_07pXvnPkSLZfykDh_7VeFxfLy74pYlzhMO8N_Q,5183
 py2docfx/convert_prepare/conf_templates/conf.py_t,sha256=VL5K6xUTIw2cb4GFjDXu5iCZFPA2Kb36KLvier78HFg,3811
 py2docfx/convert_prepare/conf_templates/master_doc.rst_t,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -23,17 +23,19 @@ py2docfx/convert_prepare/post_process/merge_toc.py,sha256=jF4u9Zb618SCSQG7u8IxCd
 py2docfx/convert_prepare/subpackage_merge/merge_root_package.py,sha256=uK96qL2asuSfo_3SZaoP8XZaUvjf5mNkr17JNbZR4Lg,1026
 py2docfx/convert_prepare/subpackage_merge/merge_toc.py,sha256=nkVqe8R0m8D6cyTYV7aIpMDXorvn4-LXfU_vIK_hJBg,1706
 py2docfx/convert_prepare/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-py2docfx/convert_prepare/tests/test_generate_document.py,sha256=h9uP_SsplHtmTHWhXPGgRGUPuxiQTkySFSa_TWiBe9g,2195
+py2docfx/convert_prepare/tests/test_generate_document.py,sha256=xyGo19FNQeyh6Xpt64a-fAY0VyMswNZgR7RAHdg1IMQ,2363
 py2docfx/convert_prepare/tests/test_get_source.py,sha256=c22JfobgbEbWWiNzBNYpZm2yDfo5LwBioUuRwft9WZE,5858
 py2docfx/convert_prepare/tests/test_pack.py,sha256=46JWMNzknIptDVs7D3CuxcmqBr_OKMmaw1br9H7wqco,4134
-py2docfx/convert_prepare/tests/test_package_info.py,sha256=Id7mBpmuQb4dGZ-wEIYBeWlgLfOFKzLO8IETlUeZ0_o,2179
+py2docfx/convert_prepare/tests/test_package_info.py,sha256=OCXoIbIDD8Bu0wMiQctMzFSkyFjfaxtCFf0wS1EHqs8,2210
 py2docfx/convert_prepare/tests/test_params.py,sha256=Kc6JnoCbC3VvZwzPac1_46USsoZy-2YG-GrwhqFa8sI,2017
 py2docfx/convert_prepare/tests/test_post_process_merge_toc.py,sha256=YKOcn4_lf4syGsAvJ9BqpdUUc3SLfK4TiOX1lpXJT_Y,885
 py2docfx/convert_prepare/tests/test_source.py,sha256=xkbYu-F6xNBlXWW0nu0YDYwc5zNj8h67Qd-3QpJ_B0M,1281
-py2docfx/convert_prepare/tests/test_sphinx_caller.py,sha256=IgZ4Pm3-SNBgGW938NqIg8XlFbnE7lJvwoCip3uBY78,2533
+py2docfx/convert_prepare/tests/test_sphinx_caller.py,sha256=RvTPo3DBecr-Z9JwU7qulnfnm2d-ipv7eqebWIbaE2Y,2532
 py2docfx/convert_prepare/tests/test_subpackage.py,sha256=n0lCcdrTE1gkmmfGE85tSBMlpOEBszZafaHXffxo3Oc,4982
-py2docfx/convert_prepare/tests/data/generate_document/dummy_source_code/boo/boo.py,sha256=zXze_R2qRsJsAB8Z1LY5ZGRHWDo-j3N6Tnw1yBy9gVs,97
-py2docfx/convert_prepare/tests/data/generate_document/dummy_source_code/foo/foo.py,sha256=i2oC5eoHl-HNW1mhLotsOwgdnVzkR0jRj4rnNjMbbCU,97
+py2docfx/convert_prepare/tests/data/generate_document/dummy-sourcecode/dummy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+py2docfx/convert_prepare/tests/data/generate_document/dummy-sourcecode/dummy/sourcecode/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+py2docfx/convert_prepare/tests/data/generate_document/dummy-sourcecode/dummy/sourcecode/boo/boo.py,sha256=zXze_R2qRsJsAB8Z1LY5ZGRHWDo-j3N6Tnw1yBy9gVs,97
+py2docfx/convert_prepare/tests/data/generate_document/dummy-sourcecode/dummy/sourcecode/foo/foo.py,sha256=i2oC5eoHl-HNW1mhLotsOwgdnVzkR0jRj4rnNjMbbCU,97
 py2docfx/convert_prepare/tests/data/get_source/mock-1/setup.py,sha256=6Zcy-AkzBIuiyU67AdApqj7nF6rNNTd8733ie0M4R-A,121
 py2docfx/convert_prepare/tests/data/pack/foo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 py2docfx/convert_prepare/tests/data/pack/foo/foo.py,sha256=nFSS0Abt9DB2jpxbrpv6UNmDiz8pgeChJoPWsFDrRQU,14
@@ -99,7 +101,7 @@ py2docfx/docfx_yaml/tests/roots/test-translator-signatures/refered_objects.py,sh
 py2docfx/docfx_yaml/tests/roots/test-writer-table/code_with_table_desc.py,sha256=J4eFvXsymgFvjnwVUY0APtUGwuxvt-AFJjTaEaQ7zMQ,574
 py2docfx/docfx_yaml/tests/roots/test-writer-table/conf.py,sha256=avcbnIOV2mlGQwhMQJZC4W6UGRBRhnq1QBxjPWlySxQ,260
 py2docfx/docfx_yaml/tests/utils/test_utils.py,sha256=N61dPDo5vuRSWlq3FIbdmalOwxGMzbKDILpRBeN58RE,3374
-py2docfx-0.1.0.dev1464944.dist-info/METADATA,sha256=xftSR1Jq8HwrCQ01Ew0fQT7QWKZHFIWLGK4QURN3vT8,601
-py2docfx-0.1.0.dev1464944.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-py2docfx-0.1.0.dev1464944.dist-info/top_level.txt,sha256=5dH2uP81dczt_qQJ38wiZ-gzoVWasfiJALWRSjdbnYU,9
-py2docfx-0.1.0.dev1464944.dist-info/RECORD,,
+py2docfx-0.1.0.dev1465062.dist-info/METADATA,sha256=8KGyc82CBXUzJW6ksUtE2BaotKp0i0m1Vl4RohYGz4s,601
+py2docfx-0.1.0.dev1465062.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
+py2docfx-0.1.0.dev1465062.dist-info/top_level.txt,sha256=5dH2uP81dczt_qQJ38wiZ-gzoVWasfiJALWRSjdbnYU,9
+py2docfx-0.1.0.dev1465062.dist-info/RECORD,,