esp-docs 2.3.0__tar.gz → 2.5.0__tar.gz

{esp_docs-2.3.0 → esp_docs-2.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: esp-docs
-Version: 2.3.0
+Version: 2.5.0
 Summary: Documentation building package used at Espressif
 Home-page: https://github.com/espressif/esp-docs
 Author: Espressif

{esp_docs-2.3.0 → esp_docs-2.5.0}/pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "2.3.0"
+version = "2.5.0"
 tag_format = "v$version"
 version_files = [
     "setup.cfg:version",

{esp_docs-2.3.0 → esp_docs-2.5.0}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = esp-docs
-version = 2.3.0
+version = 2.5.0
 author = Espressif
 author_email = marius.vikhammer@espressif.com
 description = Documentation building package used at Espressif

{esp_docs-2.3.0 → esp_docs-2.5.0}/src/esp_docs/build_docs.py

@@ -26,6 +26,7 @@
 from __future__ import print_function
 
 import argparse
+import json
 import locale
 import math
 import multiprocessing
@@ -36,6 +37,7 @@ import subprocess
 import sys
 from pathlib import Path
 from .check_docs import check_docs, format_path_for_display, style_text
+from .modified_files import parse_modified_files_arg
 from .check_lang_switch import run_lang_linkcheck
 from esp_docs.constants import TARGETS
 
@@ -89,6 +91,8 @@ def main():
     parser.add_argument('--input-docs', '-i', nargs='+', default=[''],
                         help='List of documents to build relative to the doc base folder, i.e. the language folder. Defaults to all documents')
     parser.add_argument('--fast-build', '-f', action='store_true', help='Skips including doxygen generated APIs into the Sphinx build')
+    parser.add_argument('--modified-files', nargs='+', default=[],
+                        help='List of modified files relative to the project path, used by smart build logic')
     parser.add_argument('--skip-reqs-check', action='store_true', help='Skips checking python requirements.txt found in the current directory (deprecated)')
 
     action_parsers = parser.add_subparsers(dest='action')
@@ -103,6 +107,7 @@ def main():
     action_parsers.add_parser('lang-linkcheck', help='Check if link_to_translation directives are present in RST files included in toctrees')
 
     args = parser.parse_args()
+    args.modified_files = parse_modified_files_arg(args.modified_files)
 
     global languages
     if args.language is None:
@@ -183,6 +188,7 @@ def parallel_call(args, callback):
             build_info['builders'] = args.builders
             build_info['input_docs'] = args.input_docs
             build_info['doxyfile_dir'] = args.doxyfile_dir
+            build_info['modified_files'] = args.modified_files
             build_info['project_path'] = args.project_path
 
             entries.append(build_info)
@@ -225,6 +231,7 @@ def sphinx_call(build_info, builder):
     environ = {}
     environ.update(os.environ)
     environ['BUILDDIR'] = build_info['build_dir']
+    environ['DOCS_MODIFIED_FILES'] = json.dumps(build_info['modified_files'])
 
     args = [sys.executable, '-u', '-m', 'sphinx.cmd.build',
             '-j', str(build_info['sphinx_parallel_jobs']),

{esp_docs-2.3.0 → esp_docs-2.5.0}/src/esp_docs/conf_docs.py

@@ -325,6 +325,7 @@ def setup(app):
     app.add_config_value('docs_to_build', None, 'env')
     app.add_config_value('user_setup_callback', None, 'env')
     app.add_config_value('doc_id', None, 'env')
+    app.add_config_value('modified_files', os.environ.get('DOCS_MODIFIED_FILES', '[]'), 'env')
 
     # Breathe extension variables (depend on build_dir)
     # note: we generate into xml_in and then copy_if_modified to xml dir

{esp_docs-2.3.0 → esp_docs-2.5.0}/src/esp_docs/esp_extensions/run_doxygen.py

@@ -9,6 +9,7 @@ from io import open
 from collections import defaultdict
 from dataclasses import dataclass
 
+from ..modified_files import get_modified_files, normalize_modified_file_path
 from ..util.util import copy_if_modified
 
 ALL_KINDS = [
@@ -52,6 +53,16 @@ def find_doxygen_dir(doxyfile_dir):
         return doxyfile_dir
 
 
+def should_keep_api_reference(header_path, modified_files, fast_build):
+    if not fast_build:
+        return True
+
+    if not modified_files:
+        return False
+
+    return normalize_modified_file_path(header_path) in modified_files
+
+
 def generate_doxygen(app, defines):
 
     build_dir = app.config.build_dir
@@ -150,6 +161,7 @@ def convert_api_xml_to_inc(app, doxyfiles):
     inc_directory_path = '{}/inc'.format(build_dir)
 
     fast_build = os.environ.get('DOCS_FAST_BUILD', None)
+    modified_files = get_modified_files(app.config)
 
     if not os.path.isdir(xml_directory_path):
         raise RuntimeError('Directory {} does not exist!'.format(xml_directory_path))
@@ -161,28 +173,27 @@ def convert_api_xml_to_inc(app, doxyfiles):
 
     print("Generating 'api_name.inc' files with Doxygen directives")
     for api_path in api_paths:
-        rst_output = generate_directives(app, api_path.header_path, api_path.xml_file_path)
+        final_rst_output = generate_directives(app, api_path.header_path, api_path.xml_file_path)
+
+        # For fast builds we skip unchanged API reference includes, but still keep
+        # the ones generated from modified headers.
+        if not should_keep_api_reference(api_path.header_path, modified_files, fast_build):
+            final_rst_output = ''
 
         # Create subfolders if needed
         dir_name = os.path.dirname(api_path.inc_file_path)
         if dir_name:
             os.makedirs(dir_name, exist_ok=True)
 
+        inc_exists = os.path.isfile(api_path.inc_file_path)
         previous_rst_output = ''
-        if os.path.isfile(api_path.inc_file_path):
+        if inc_exists:
             with open(api_path.inc_file_path, 'r', encoding='utf-8') as inc_file_old:
                 previous_rst_output = inc_file_old.read()
 
-        if previous_rst_output != rst_output:
-            with open(api_path.inc_file_path, 'w', encoding='utf-8') as inc_file:
-                inc_file.write(rst_output)
-
-        # For fast builds we wipe the doxygen api documentation.
-        # Parsing this output during the sphinx build process is
-        # what takes 95% of the build time
-        if fast_build:
+        if (not inc_exists) or previous_rst_output != final_rst_output:
             with open(api_path.inc_file_path, 'w', encoding='utf-8') as inc_file:
-                inc_file.write('')
+                inc_file.write(final_rst_output)
 
 
 def get_doxyfile_input_paths(app, doxyfile_path):
@@ -215,7 +226,7 @@ def get_doxyfile_input_paths(app, doxyfile_path):
             # process only lines that are not comments
             if line.find('#') == -1:
                 # extract header file path inside project folder
-                m = re.search('\(PROJECT_PATH\)/(.*\.hp*)', line)  # noqa: W605 - regular expression
+                m = re.search(r'\(PROJECT_PATH\)/(.*\.hp*)', line)
                 if m is None:
                     raise ValueError("Doxygen input statements should be specified using $(PROJECT_PATH) env variable, instead got {}".format(line))
                 header_file_path = m.group(1)

esp_docs-2.5.0/src/esp_docs/modified_files.py

@@ -0,0 +1,53 @@
+from __future__ import unicode_literals
+
+import json
+import os
+
+
+def parse_modified_files_arg(modified_files):
+    if not modified_files:
+        return []
+
+    normalized_paths = []
+    for value in modified_files:
+        if not value:
+            continue
+
+        normalized_paths.extend(
+            path for path in value.split(';') if path
+        )
+
+    return normalized_paths
+
+
+def normalize_modified_file_path(file_path, project_path=None):
+    normalized_path = file_path
+
+    if project_path and os.path.isabs(normalized_path):
+        normalized_path = os.path.relpath(normalized_path, project_path)
+
+    normalized_path = os.path.normpath(normalized_path).replace('\\', '/')
+
+    if normalized_path == '.':
+        return ''
+
+    while normalized_path.startswith('./'):
+        normalized_path = normalized_path[2:]
+
+    return normalized_path
+
+
+def get_modified_files(config):
+    modified_files = config.modified_files
+
+    if not modified_files:
+        return set()
+
+    if isinstance(modified_files, str):
+        modified_files = json.loads(modified_files)
+
+    return {
+        normalize_modified_file_path(path, config.project_path)
+        for path in modified_files
+        if path
+    }

{esp_docs-2.3.0 → esp_docs-2.5.0}/src/esp_docs/upload_docs_s3.py

@@ -53,10 +53,14 @@ Optional environment variables
 """
 
 import glob
+import functools
 import mimetypes
 import os
 import os.path
 import shutil
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from urllib.parse import quote
 
 import boto3
 from botocore.config import Config
@@ -65,6 +69,23 @@ from .util.util import is_stable_version, env
 from .sanitize_version import sanitize_version  # noqa
 
 
+def print_timing(label, started_at):
+    elapsed = time.perf_counter() - started_at
+    print(f'[timing] {label} took {elapsed:.3f}s', flush=True)
+
+
+def timed(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        started_at = time.perf_counter()
+        try:
+            return func(*args, **kwargs)
+        finally:
+            print_timing(func.__name__, started_at)
+
+    return wrapper
+
+
 class S3Config:
     alias = 'deploy-docs-s3'
     access_key: str = env('DOCS_S3_ACCESS_KEY_ID')
@@ -82,6 +103,7 @@ class S3Config:
     )
 
 
+@timed
 def main():
     # if you get KeyErrors on the following lines, it's probably because you're not running in Gitlab CI
     git_ver = env('GIT_VER')  # output of git describe --always
@@ -102,7 +124,9 @@ def main():
     docs_path = f'{commit_sha}/{S3Config.docs_path}'
 
     tmp_path = copy_docs_to_tmp_folder(version, build_dir)
-    upload(tmp_path, docs_path)
+    uploaded_url = upload(tmp_path, docs_path)
+    if uploaded_url:
+        print(f'Docs uploaded. Example file: {uploaded_url}')
 
     # note: it would be neater to use symlinks for stable, but because of the directory order
     # (language first) it's kind of a pain to do on a remote server, so we just repeat the
@@ -111,34 +135,75 @@ def main():
     if is_stable_version(version) and do_deploy_stable:
         print('Deploying again as stable version...')
         tmp_path = copy_docs_to_tmp_folder('stable', build_dir)
-        upload(tmp_path, docs_path)
+        uploaded_url = upload(tmp_path, docs_path)
+        if uploaded_url:
+            print(f'Stable docs uploaded. Example file: {uploaded_url}')
 
 
 def _upload_one(local_path, key_name):
+    started_at = time.perf_counter()
     content_type, encoding = mimetypes.guess_type(local_path)
 
     extra_args = {}
     if content_type:
         extra_args['ContentType'] = content_type
 
-    S3Config.s3.upload_file(
-        local_path,
-        S3Config.bucket_name,
-        key_name,
-        ExtraArgs=extra_args if extra_args else None,
-    )
+    try:
+        S3Config.s3.upload_file(
+            local_path,
+            S3Config.bucket_name,
+            key_name,
+            ExtraArgs=extra_args if extra_args else None,
+        )
+    finally:
+        print_timing(f'_upload_one: {local_path}', started_at)
 
 
+@timed
 def upload(source_dir, remote_dir):
     prefix = remote_dir.strip('/')
+    example_key_name = None
+    upload_jobs = []
+
     for root, _, files in os.walk(source_dir):
         for filename in files:
             local_path = str(os.path.join(root, filename))
             rel_path = os.path.relpath(local_path, source_dir).replace(os.sep, '/')
             key_name = f'{prefix}/{rel_path}'
-            _upload_one(local_path, key_name)
+            upload_jobs.append((local_path, key_name))
+
+            if filename == 'index.html' and example_key_name is None:
+                example_key_name = key_name
+
+    if not upload_jobs:
+        return build_s3_file_url(example_key_name) if example_key_name else None
+
+    max_workers = max(1, int(os.getenv('DOCS_S3_UPLOAD_WORKERS', 16)))
+    print(f'Uploading {len(upload_jobs)} files with {max_workers} workers...')
+
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = [
+            executor.submit(_upload_one, local_path, key_name)
+            for local_path, key_name in upload_jobs
+        ]
+        try:
+            for future in as_completed(futures):
+                future.result()
+        except Exception:
+            for future in futures:
+                future.cancel()
+            raise
+
+    return build_s3_file_url(example_key_name) if example_key_name else None
+
+
+def build_s3_file_url(key_name):
+    key_path = quote(key_name.lstrip('/'), safe='/')
+    endpoint = S3Config.endpoint.removesuffix('/')
+    return f'{endpoint.replace("://", f"://{S3Config.bucket_name}.", 1)}/{key_path}'
 
 
+@timed
 def copy_docs_to_tmp_folder(version, build_dir):
     """Copy all docs to a tmp folder, maintaining the directory structure used to deploy as
     the given version"""

{esp_docs-2.3.0 → esp_docs-2.5.0}/src/esp_docs.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: esp-docs
-Version: 2.3.0
+Version: 2.5.0
 Summary: Documentation building package used at Espressif
 Home-page: https://github.com/espressif/esp-docs
 Author: Espressif

{esp_docs-2.3.0 → esp_docs-2.5.0}/src/esp_docs.egg-info/SOURCES.txt

@@ -11,6 +11,7 @@ src/esp_docs/constants.py
 src/esp_docs/deploy_docs.py
 src/esp_docs/deploy_docs_s3.py
 src/esp_docs/get_github_rev.py
+src/esp_docs/modified_files.py
 src/esp_docs/sanitize_version.py
 src/esp_docs/upload_docs_s3.py
 src/esp_docs.egg-info/PKG-INFO