azdev 0.1.90__tar.gz → 0.1.93__tar.gz

{azdev-0.1.90 → azdev-0.1.93}/HISTORY.rst

@@ -2,6 +2,19 @@
 
 Release History
 ===============
+0.1.93
+++++++
+* `azdev linter`: Fix `None` path for added files in `git diff` for `missing_command_example` rule
+
+0.1.92
+++++++
+* `azdev linter`: Add `missing_command_example` rule
+* `azdev linter`: Set `disallowed_html_tags` and `broken_site_link` detection in `high` severity to block future format issues
+
+0.1.91
+++++++
+* `azdev generate-breaking-change-report`: Update report report template.
+
 0.1.90
 ++++++
 * `azdev cmdcov`: Fix incorrect detection of code changes as new commands

{azdev-0.1.90/azdev.egg-info → azdev-0.1.93}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: azdev
-Version: 0.1.90
+Version: 0.1.93
 Summary: Microsoft Azure CLI Developer Tools
 Home-page: https://github.com/Azure/azure-cli-dev-tools
 Author: Microsoft Corporation
@@ -148,6 +148,19 @@ License
 
 Release History
 ===============
+0.1.93
+++++++
+* `azdev linter`: Fix `None` path for added files in `git diff` for `missing_command_example` rule
+
+0.1.92
+++++++
+* `azdev linter`: Add `missing_command_example` rule
+* `azdev linter`: Set `disallowed_html_tags` and `broken_site_link` detection in `high` severity to block future format issues
+
+0.1.91
+++++++
+* `azdev generate-breaking-change-report`: Update report report template.
+
 0.1.90
 ++++++
 * `azdev cmdcov`: Fix incorrect detection of code changes as new commands

{azdev-0.1.90 → azdev-0.1.93}/azdev/__init__.py

@@ -4,4 +4,4 @@
 # license information.
 # -----------------------------------------------------------------------------
 
-__VERSION__ = '0.1.90'
+__VERSION__ = '0.1.93'

{azdev-0.1.90 → azdev-0.1.93}/azdev/operations/breaking_change/__init__.py

@@ -25,6 +25,15 @@ class BreakingChangeItem:
         self.command = command
         self.detail = detail
         self.target_version = target_version
+        self.group_ref = None
+
+    def calc_ref(self, loader):
+        if not loader:
+            return
+        if self.command in loader.command_group_table:
+            self.group_ref = self.command.split()
+        else:
+            self.group_ref = self.command.split()[:-1]
 
 
 def _load_commands():
@@ -247,7 +256,9 @@ def _handle_upcoming_breaking_changes(selected_mod_names, source):
         yield from _handle_core(source)
 
     for module, loader in _iter_and_prepare_module_loader(command_loader, selected_mod_names):
-        yield from _handle_module(module, loader, source)
+        for bc_item in _handle_module(module, loader, source):
+            bc_item.calc_ref(loader)
+            yield bc_item
 
 
 def _filter_breaking_changes(iterator, max_version=None):
@@ -276,18 +287,19 @@ def _group_breaking_change_items(iterator, group_by_version=False):
     if group_by_version:
         upcoming_breaking_changes = defaultdict(    # module to command
             lambda: defaultdict(    # command to version
-                lambda: defaultdict(    # version to list of breaking changes
-                    lambda: [])))
+                lambda: {'group_ref': None, 'items': defaultdict(    # version to list of breaking changes
+                    lambda: [])}))
     else:
         upcoming_breaking_changes = defaultdict(    # module to command
             lambda: defaultdict(    # command to list of breaking changes
-                lambda: []))
+                lambda: {'group_ref': None, 'items': []}))
     for item in iterator:
         version = item.target_version if item.target_version else 'Unspecific'
+        upcoming_breaking_changes[item.module][item.command]['group_ref'] = item.group_ref
         if group_by_version:
-            upcoming_breaking_changes[item.module][item.command][version].append(item.detail)
+            upcoming_breaking_changes[item.module][item.command]['items'][version].append(item.detail)
         else:
-            upcoming_breaking_changes[item.module][item.command].append(item.detail)
+            upcoming_breaking_changes[item.module][item.command]['items'].append(item.detail)
     return upcoming_breaking_changes
 
 

azdev-0.1.93/azdev/operations/breaking_change/markdown_template.jinja2

@@ -0,0 +1,37 @@
+# Upcoming breaking changes in Azure CLI
+
+The breaking changes listed in this article are planned for the next major release of the Azure CLI unless otherwise noted. Per our [Support lifecycle](./azure-cli-support-lifecycle.md), breaking changes in Azure CLI Core reference groups occur twice a year.
+
+{% for module, command_bc in module_bc.items() -%}
+## {{ module }}
+
+{% for command, multi_version_bcs in command_bc.items() -%}
+{% if not (module == 'core' and command == 'core') -%}
+### `{{ command }}`
+
+{% endif -%}
+{% if multi_version_bcs.group_ref -%}
+[Link to {{ multi_version_bcs.group_ref|join(' ') }} reference group](/cli/azure/{{ multi_version_bcs.group_ref|join('/') }})
+
+{% endif -%}
+{% if multi_version_bcs['items'] is mapping -%}
+{% for version, bcs in multi_version_bcs['items'] | dictsort -%}
+###{%- if not (module == 'core' and command == 'core') -%}#{%- endif %} Deprecated in {{ version }}
+
+{% for bc in bcs -%}
+- {{ bc.detail }}
+{% endfor %}
+
+{% endfor -%}
+{% else -%}
+
+{% for bc in multi_version_bcs['items'] -%}
+- {{ bc }}
+{% endfor %}
+
+{% endif -%}
+{% endfor -%}
+{% endfor -%}
+
+> [!NOTE]
+> This article provides information on upcoming breaking changes. For previously published breaking changes, see [Azure CLI release notes](./release-notes-azure-cli.md).

{azdev-0.1.90 → azdev-0.1.93}/azdev/operations/constant.py

@@ -4,7 +4,7 @@
 # license information.
 # -----------------------------------------------------------------------------
 # pylint: disable=line-too-long
-
+import os
 ENCODING = 'utf-8'
 
 # Base on https://github.com/Azure/azure-cli/blob/dev/.github/CODEOWNERS
@@ -259,3 +259,8 @@ VERSION_PREVIEW_TAG = "preview"
 PREVIEW_INIT_SUFFIX = "b1"
 
 CLI_EXTENSION_INDEX_URL = "https://azcliextensionsync.blob.core.windows.net/index1/index.json"
+
+CMD_EXAMPLE_CONFIG_FILE = "./data/cmd_example_config.json"
+CMD_EXAMPLE_CONFIG_FILE_PATH = f"{os.path.dirname(os.path.realpath(__file__))}/linter/{CMD_EXAMPLE_CONFIG_FILE}"
+CMD_EXAMPLE_CONFIG_FILE_URL = "https://azcmdchangemgmt.blob.core.windows.net/azure-cli-dev-tool-config/cmd_example_config.json"
+CMD_EXAMPLE_DEFAULT = 1

azdev-0.1.93/azdev/operations/linter/data/cmd_example_config.json

@@ -0,0 +1,10 @@
+{
+  "create": 1,
+  "add": 1,
+  "update": 1,
+  "list": 0,
+  "delete": 0,
+  "remove": 0,
+  "show": 0,
+  "wait": 0
+}

{azdev-0.1.90 → azdev-0.1.93}/azdev/operations/linter/linter.py

@@ -3,7 +3,7 @@
 # Licensed under the MIT License. See License.txt in the project root for
 # license information.
 # -----------------------------------------------------------------------------
-
+# pylint: disable=line-too-long
 from difflib import context_diff
 from enum import Enum
 from importlib import import_module
@@ -17,6 +17,7 @@ from knack.log import get_logger
 
 from azdev.operations.regex import (
     get_all_tested_commands_from_regex,
+    search_aaz_raw_command, search_aaz_custom_command,
     search_argument,
     search_argument_context,
     search_command,
@@ -24,7 +25,8 @@ from azdev.operations.regex import (
     search_command_group)
 from azdev.utilities import diff_branches_detail, diff_branch_file_patch
 from azdev.utilities.path import get_cli_repo_path, get_ext_repo_paths
-from .util import share_element, exclude_commands, LinterError
+from .util import (share_element, exclude_commands, LinterError, get_cmd_example_configurations,
+                   get_cmd_example_threshold)
 
 PACKAGE_NAME = 'azdev.operations.linter'
 _logger = get_logger(__name__)
@@ -224,6 +226,31 @@ class Linter:  # pylint: disable=too-many-public-methods, too-many-instance-attr
         all_tested_command = self._detect_tested_command(diff_index)
         return self._run_parameter_test_coverage(parameters, all_tested_command)
 
+    def check_missing_command_example(self):
+        _exclude_commands = self._get_cmd_exclusions(rule_name="missing_command_example")
+        cmd_example_config = get_cmd_example_configurations()
+        commands = self._detect_modified_command()
+        violations = []
+        for cmd in commands:
+            if cmd in _exclude_commands:
+                continue
+            cmd_help = self._loaded_help.get(cmd, None)
+            if not cmd_help:
+                continue
+            cmd_suffix = cmd.split()[-1]
+            cmd_example_threshold = get_cmd_example_threshold(cmd_suffix, cmd_example_config)
+            if cmd_example_threshold == 0:
+                continue
+            if not hasattr(cmd_help, "parameters") or len(cmd_help.parameters) == 0:
+                # skip cmd without parameters
+                continue
+            if not hasattr(cmd_help, "examples") or len(cmd_help.examples) < cmd_example_threshold:
+                violations.append(f'Command `{cmd}` should have at least {cmd_example_threshold} example(s)')
+        if violations:
+            violations.insert(0, 'Check command example failed.')
+            violations.extend(['Please add examples for the modified command or add the command in rule_exclusions: missing_command_example in linter_exclusions.yml'])
+        return violations
+
     def _get_exclusions(self):
         _exclude_commands = set()
         _exclude_parameters = set()
@@ -238,6 +265,16 @@ class Linter:  # pylint: disable=too-many-public-methods, too-many-instance-attr
         _logger.debug('exclude_comands: %s', _exclude_commands)
         return _exclude_commands, _exclude_parameters
 
+    def _get_cmd_exclusions(self, rule_name=None):
+        _exclude_commands = set()
+        if not rule_name:
+            return _exclude_commands
+        for command, details in self.exclusions.items():
+            if 'rule_exclusions' in details and rule_name in details['rule_exclusions']:
+                _exclude_commands.add(command)
+        _logger.debug('exclude_commands: %s', _exclude_commands)
+        return _exclude_commands
+
     def _split_path(self, path: str):
         parts = path.rsplit('/', maxsplit=1)
         return parts if len(parts) == 2 else ('', parts[0])
@@ -387,11 +424,49 @@ class Linter:  # pylint: disable=too-many-public-methods, too-many-instance-attr
                 'Or add the parameter with missing_parameter_test_coverage rule in linter_exclusions.yml'])
         return exec_state, violations
 
+    def _detect_modified_command(self):
+        modified_commands = set()
+        diff_patches = diff_branch_file_patch(repo=self.git_repo, target=self.git_target, source=self.git_source)
+        for change in diff_patches:
+            if not change.b_path or not change.diff:
+                continue
+            file_path, filename = self._split_path(change.b_path)
+            if "commands.py" not in filename and "aaz" not in file_path:
+                continue
+            current_lines = self._read_blob_lines(change.b_blob)
+            patch = change.diff.decode("utf-8")
+            patch_lines = patch.splitlines()
+            if 'commands.py' in filename:
+                added_lines = [line for line in patch_lines if line.startswith('+') and not line.startswith('+++')]
+                for line in added_lines:
+                    if aaz_custom_command := search_aaz_custom_command(line):
+                        modified_commands.add(aaz_custom_command)
+
+                for row_num, line in enumerate(patch_lines):
+                    if not line.startswith("+") or line.startswith('+++'):
+                        continue
+                    manual_command_suffix = search_command(line)
+                    if manual_command_suffix:
+                        idx = self._get_line_number(patch_lines, row_num, r'@@ -(\d+),(?:\d+) \+(?:\d+),(?:\d+) @@')
+                        manual_command = search_command_group(idx, current_lines, manual_command_suffix)
+                        if manual_command:
+                            modified_commands.add(manual_command)
+
+            if "aaz" in file_path:
+                if aaz_raw_command := search_aaz_raw_command(patch):
+                    modified_commands.add(aaz_raw_command)
+
+        commands = list(modified_commands)
+        _logger.debug('Modified commands: %s', modified_commands)
+        return commands
+
     def _get_diffed_patches(self):
         if not self.git_source or not self.git_target or not self.git_repo:
             return
         diff_patches = diff_branch_file_patch(repo=self.git_repo, target=self.git_target, source=self.git_source)
         for change in diff_patches:
+            if not change.diff:
+                continue
             patch = change.diff.decode("utf-8")
             added_lines = [line for line in patch.splitlines() if line.startswith('+') and not line.startswith('+++')]
             self.diffed_lines |= set(added_lines)

{azdev-0.1.90 → azdev-0.1.93}/azdev/operations/linter/rules/command_coverage_rules.py

@@ -22,3 +22,11 @@ def missing_parameter_test_coverage(linter):
     if not exec_state:
         violation_msg = "\n\t".join(violations)
         raise RuleError(violation_msg + "\n")
+
+
+@CommandCoverageRule(LinterSeverity.HIGH)
+def missing_command_example(linter):
+    violations = linter.check_missing_command_example()
+    if violations:
+        violation_msg = "\n\t".join(violations)
+        raise RuleError(violation_msg + "\n")

{azdev-0.1.90 → azdev-0.1.93}/azdev/operations/linter/rules/command_group_rules.py

@@ -45,7 +45,7 @@ def require_wait_command_if_no_wait(linter, command_group_name):
             raise RuleError("Group does not have a 'wait' command, yet '{}' exposes '--no-wait'".format(cmd))
 
 
-@CommandGroupRule(LinterSeverity.MEDIUM)
+@CommandGroupRule(LinterSeverity.HIGH)
 def disallowed_html_tag_from_command_group(linter, command_group_name):
     if command_group_name == '' or not linter.get_loaded_help_entry(command_group_name):
         return
@@ -69,9 +69,11 @@ def broken_site_link_from_command_group(linter, command_group_name):
     if command_group_name == '' or not linter.get_loaded_help_entry(command_group_name):
         return
     help_entry = linter.get_loaded_help_entry(command_group_name)
-    if help_entry.short_summary and (broken_links := has_broken_site_links(help_entry.short_summary)):
+    if help_entry.short_summary and (broken_links := has_broken_site_links(help_entry.short_summary,
+                                                                           linter.diffed_lines)):
         raise RuleError("Broken links {} in short summary. "
                         "If link is an example, please wrap it with backtick. ".format(broken_links))
-    if help_entry.long_summary and (broken_links := has_broken_site_links(help_entry.long_summary)):
+    if help_entry.long_summary and (broken_links := has_broken_site_links(help_entry.long_summary,
+                                                                          linter.diffed_lines)):
         raise RuleError("Broken links {} in long summary. "
                         "If link is an example, please wrap it with backtick. ".format(broken_links))

{azdev-0.1.90 → azdev-0.1.93}/azdev/operations/linter/rules/command_rules.py

@@ -40,7 +40,7 @@ def group_delete_commands_should_confirm(linter, command_name):
                             "Please make sure to ask for confirmation.")
 
 
-@CommandRule(LinterSeverity.MEDIUM)
+@CommandRule(LinterSeverity.HIGH)
 def disallowed_html_tag_from_command(linter, command_name):
     if command_name == '' or not linter.get_loaded_help_entry(command_name):
         return
@@ -64,9 +64,11 @@ def broken_site_link_from_command(linter, command_name):
     if command_name == '' or not linter.get_loaded_help_entry(command_name):
         return
     help_entry = linter.get_loaded_help_entry(command_name)
-    if help_entry.short_summary and (broken_links := has_broken_site_links(help_entry.short_summary)):
+    if help_entry.short_summary and (broken_links := has_broken_site_links(help_entry.short_summary,
+                                                                           linter.diffed_lines)):
         raise RuleError("Broken links {} in short summary. "
                         "If link is an example, please wrap it with backtick. ".format(broken_links))
-    if help_entry.long_summary and (broken_links := has_broken_site_links(help_entry.long_summary)):
+    if help_entry.long_summary and (broken_links := has_broken_site_links(help_entry.long_summary,
+                                                                          linter.diffed_lines)):
         raise RuleError("Broken links {} in long summary. "
                         "If link is an example, please wrap it with backtick. ".format(broken_links))

{azdev-0.1.90 → azdev-0.1.93}/azdev/operations/linter/rules/parameter_rules.py

@@ -174,7 +174,7 @@ def option_should_not_contain_under_score(linter, command_name, parameter_name):
             raise RuleError("Argument's option {} contains '_' which should be '-' instead.".format(option))
 
 
-@ParameterRule(LinterSeverity.MEDIUM)
+@ParameterRule(LinterSeverity.HIGH)
 def disallowed_html_tag_from_parameter(linter, command_name, parameter_name):
     if linter.command_expired(command_name) or not linter.get_parameter_help_info(command_name, parameter_name):
         return
@@ -199,9 +199,11 @@ def broken_site_link_from_parameter(linter, command_name, parameter_name):
     if linter.command_expired(command_name) or not linter.get_parameter_help_info(command_name, parameter_name):
         return
     help_entry = linter.get_parameter_help_info(command_name, parameter_name)
-    if help_entry.short_summary and (broken_links := has_broken_site_links(help_entry.short_summary)):
+    if help_entry.short_summary and (broken_links := has_broken_site_links(help_entry.short_summary,
+                                                                           linter.diffed_lines)):
         raise RuleError("Broken links {} in short summary. "
                         "If link is an example, please wrap it with backtick. ".format(broken_links))
-    if help_entry.long_summary and (broken_links := has_broken_site_links(help_entry.long_summary)):
+    if help_entry.long_summary and (broken_links := has_broken_site_links(help_entry.long_summary,
+                                                                          linter.diffed_lines)):
         raise RuleError("Broken links {} in long summary. "
                         "If link is an example, please wrap it with backtick. ".format(broken_links))

{azdev-0.1.90 → azdev-0.1.93}/azdev/operations/linter/util.py

@@ -6,12 +6,15 @@
 
 import copy
 import re
+import os
+import json
 import requests
 
 from knack.log import get_logger
 
 from azdev.utilities import get_name_index
-from azdev.operations.constant import ALLOWED_HTML_TAG
+from azdev.operations.constant import (ALLOWED_HTML_TAG, CMD_EXAMPLE_CONFIG_FILE_URL,
+                                       CMD_EXAMPLE_CONFIG_FILE_PATH, CMD_EXAMPLE_DEFAULT)
 
 
 logger = get_logger(__name__)
@@ -155,3 +158,26 @@ def has_broken_site_links(help_message, filtered_lines=None):
     if filtered_lines:
         invalid_urls = [s for s in invalid_urls if any(s in diff_line for diff_line in filtered_lines)]
     return invalid_urls
+
+
+def get_cmd_example_configurations():
+    cmd_example_threshold = {}
+    remote_res = requests.get(CMD_EXAMPLE_CONFIG_FILE_URL)
+    if remote_res.status_code != 200:
+        logger.warning("remote cmd example configuration fetch error, use local dict")
+        if not os.path.exists(CMD_EXAMPLE_CONFIG_FILE_PATH):
+            logger.info("cmd_example_config.json not exist, skipped")
+            return cmd_example_threshold
+        with open(CMD_EXAMPLE_CONFIG_FILE_PATH, "r") as f_in:
+            cmd_example_threshold = json.load(f_in)
+    else:
+        logger.info("remote cmd example configuration fetch success")
+        cmd_example_threshold = remote_res.json()
+    return cmd_example_threshold
+
+
+def get_cmd_example_threshold(cmd_suffix, cmd_example_config):
+    for cmd_type, threshold in cmd_example_config.items():
+        if cmd_suffix.find(cmd_type) != -1:
+            return threshold
+    return CMD_EXAMPLE_DEFAULT

{azdev-0.1.90 → azdev-0.1.93}/azdev/operations/regex.py

@@ -173,3 +173,31 @@ def search_deleted_command(line):
     if ref:
         command = ref[0].split(',')[0].strip("'")
     return command
+
+
+def search_aaz_custom_command(line):
+    """
+    re match pattern
+    +   self.command_table['monitor autoscale update'] = AutoScaleUpdate(loader=self)
+    """
+    cmd = ''
+    aaz_custom_cmd_pattern = r"\+.*\.command_table\[['\"](.*?)['\"]\]"
+    ref = re.findall(aaz_custom_cmd_pattern, line)
+    if ref:
+        cmd = ref[0].strip()
+    return cmd
+
+
+def search_aaz_raw_command(lines):
+    """
+    re match pattern
+    +@register_command(
+    +      "monitor autoscale update",
+    +)
+    """
+    cmd = ''
+    aaz_raw_cmd_pattern = r"\+@register_command\([\s\S]*?\+.*?['\"](.*?)['\"]"
+    ref = re.findall(aaz_raw_cmd_pattern, str(lines))
+    if ref:
+        cmd = ref[0].strip()
+    return cmd

{azdev-0.1.90 → azdev-0.1.93/azdev.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: azdev
-Version: 0.1.90
+Version: 0.1.93
 Summary: Microsoft Azure CLI Developer Tools
 Home-page: https://github.com/Azure/azure-cli-dev-tools
 Author: Microsoft Corporation
@@ -148,6 +148,19 @@ License
 
 Release History
 ===============
+0.1.93
+++++++
+* `azdev linter`: Fix `None` path for added files in `git diff` for `missing_command_example` rule
+
+0.1.92
+++++++
+* `azdev linter`: Add `missing_command_example` rule
+* `azdev linter`: Set `disallowed_html_tags` and `broken_site_link` detection in `high` severity to block future format issues
+
+0.1.91
+++++++
+* `azdev generate-breaking-change-report`: Update report report template.
+
 0.1.90
 ++++++
 * `azdev cmdcov`: Fix incorrect detection of code changes as new commands

{azdev-0.1.90 → azdev-0.1.93}/azdev.egg-info/SOURCES.txt

@@ -75,6 +75,7 @@ azdev/operations/linter/__init__.py
 azdev/operations/linter/linter.py
 azdev/operations/linter/rule_decorators.py
 azdev/operations/linter/util.py
+azdev/operations/linter/data/cmd_example_config.json
 azdev/operations/linter/pylint_checkers/__init__.py
 azdev/operations/linter/pylint_checkers/show_command.py
 azdev/operations/linter/rules/__init__.py

{azdev-0.1.90 → azdev-0.1.93}/setup.py

@@ -93,6 +93,7 @@ setup(
         'azdev.config': ['*.*', 'cli_pylintrc', 'ext_pylintrc'],
         'azdev.mod_templates': ['*.*'],
         'azdev.operations.linter.rules': ['ci_exclusions.yml'],
+        'azdev.operations.linter': ["data/*"],
         'azdev.operations.cmdcov': ['*.*'],
         'azdev.operations.breaking_change': ['*.*'],
     },

azdev-0.1.90/azdev/operations/breaking_change/markdown_template.jinja2

@@ -1,28 +0,0 @@
-# Upcoming breaking changes in Azure CLI
-
-{% for module, command_bc in module_bc.items() -%}
-## {{ module }}
-
-{% for command, multi_version_bcs in command_bc.items() -%}
-{% if not (module == 'core' and command == 'core') -%}
-### `{{ command }}`
-
-{% endif -%}
-{% if multi_version_bcs is mapping -%}
-{% for version, bcs in multi_version_bcs | dictsort -%}
-###{%- if not (module == 'core' and command == 'core') -%}#{%- endif %} Deprecated in {{ version }}
-
-{% for bc in bcs -%}
-- {{ bc }}
-{% endfor %}
-
-{% endfor -%}
-{% else -%}
-
-{% for bc in multi_version_bcs -%}
-- {{ bc }}
-{% endfor %}
-
-{% endif -%}
-{% endfor -%}
-{% endfor -%}