azdev 0.1.83__tar.gz → 0.1.85__tar.gz

{azdev-0.1.83 → azdev-0.1.85}/HISTORY.rst

@@ -2,6 +2,15 @@
 
 Release History
 ===============
+0.1.85
+++++++
+* `azdev setup`: Setup the upstream and enable .githooks for azure-cli and azure-cli-extensions repos
+* `azdev linter`: Add `disallowed_html_tags` and `broken_site_link` detection in linter rule and set them as `Medium` for unblock CI pipeline temporarily
+
+0.1.84
+++++++
+* `azdev generate-breaking-change-report`: Fix `azdev -h` error caused by global importing `azure.cli.core` in `breaking-change.py` module.
+
 0.1.83
 ++++++
 * `azdev generate-breaking-change-report`: Fix `azdev.operations.breaking_change` not included in `setup.py`.

{azdev-0.1.83/azdev.egg-info → azdev-0.1.85}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: azdev
-Version: 0.1.83
+Version: 0.1.85
 Summary: Microsoft Azure CLI Developer Tools
 Home-page: https://github.com/Azure/azure-cli-dev-tools
 Author: Microsoft Corporation
@@ -148,6 +148,15 @@ License
 
 Release History
 ===============
+0.1.85
+++++++
+* `azdev setup`: Setup the upstream and enable .githooks for azure-cli and azure-cli-extensions repos
+* `azdev linter`: Add `disallowed_html_tags` and `broken_site_link` detection in linter rule and set them as `Medium` for unblock CI pipeline temporarily
+
+0.1.84
+++++++
+* `azdev generate-breaking-change-report`: Fix `azdev -h` error caused by global importing `azure.cli.core` in `breaking-change.py` module.
+
 0.1.83
 ++++++
 * `azdev generate-breaking-change-report`: Fix `azdev.operations.breaking_change` not included in `setup.py`.

{azdev-0.1.83 → azdev-0.1.85}/azdev/__init__.py

@@ -4,4 +4,4 @@
 # license information.
 # -----------------------------------------------------------------------------
 
-__VERSION__ = '0.1.83'
+__VERSION__ = '0.1.85'

{azdev-0.1.83 → azdev-0.1.85}/azdev/help.py

@@ -160,7 +160,7 @@ helps['linter'] = """
           text: azdev linter --repo azure-cli --tgt upstream/master --src upstream/dev
 """
 
-helps['scan'] = """
+helps['scan'] = r"""
     short-summary: Scan secrets for files or string
     long-summary: Check built-in scanning rules at https://github.com/microsoft/security-utilities/blob/main/GeneratedRegexPatterns/PreciselyClassifiedSecurityKeys.json
     examples:

{azdev-0.1.83 → azdev-0.1.85}/azdev/operations/breaking_change/__init__.py

@@ -9,8 +9,6 @@ from collections import defaultdict
 from importlib import import_module
 
 import packaging.version
-from azure.cli.core.breaking_change import MergedStatusTag, UpcomingBreakingChangeTag, TargetVersion
-from knack.deprecation import Deprecated
 from knack.log import get_logger
 
 from azdev.operations.statistics import _create_invoker_and_load_cmds  # pylint: disable=protected-access
@@ -82,6 +80,9 @@ def _handle_custom_breaking_change(module, command, breaking_change):
 
 
 def _handle_status_tag(module, command, status_tag):
+    from knack.deprecation import Deprecated
+    from azure.cli.core.breaking_change import MergedStatusTag, UpcomingBreakingChangeTag, TargetVersion
+
     if isinstance(status_tag, MergedStatusTag):
         for tag in status_tag.tags:
             yield from _handle_status_tag(module, command, tag)
@@ -107,6 +108,8 @@ def _handle_command_deprecation(module, command, deprecate_info):
 
 
 def _calc_target_of_arg_deprecation(arg_name, arg_settings):
+    from knack.deprecation import Deprecated
+
     option_str_list = []
     depr = arg_settings.get('deprecate_info')
     for option in arg_settings.get('option_list', []):
@@ -128,6 +131,8 @@ def _handle_arg_deprecation(module, command, target, deprecation_info):
 
 
 def _handle_options_deprecation(module, command, options):
+    from knack.deprecation import Deprecated
+
     deprecate_option_map = defaultdict(lambda: [])
     for option in options:
         if isinstance(option, Deprecated):

{azdev-0.1.83 → azdev-0.1.85}/azdev/operations/cmdcov/cmdcov.py

@@ -239,8 +239,8 @@ class CmdcovManager:
             self.command_test_coverage['Total'][0] += count
             self.command_test_coverage['Total'][1] += len(self.all_untested_commands[module])
         self.command_test_coverage['Total'][2] = f'''{self.command_test_coverage["Total"][0] /
-                                                 (self.command_test_coverage["Total"][0] +
-                                                  self.command_test_coverage["Total"][1]):.3%}'''
+                                                      (self.command_test_coverage["Total"][0] +
+                                                       self.command_test_coverage["Total"][1]):.3%}'''
         logger.warning(self.command_test_coverage)
         return self.command_test_coverage
 

{azdev-0.1.83 → azdev-0.1.85}/azdev/operations/constant.py

@@ -3,6 +3,7 @@
 # Licensed under the MIT License. See License.txt in the project root for
 # license information.
 # -----------------------------------------------------------------------------
+# pylint: disable=line-too-long
 
 ENCODING = 'utf-8'
 
@@ -79,6 +80,23 @@ EXCLUDE_MODULES = [
     'util'
 ]
 
+# refer to doc: https://review.learn.microsoft.com/en-us/help/platform/metadata-taxonomies/allowed-html?branch=main
+ALLOWED_HTML_TAG = [
+    "a", "address", "article", "b", "blockquote", "br", "button", "br /",
+    "caption", "center", "cite", "code", "col", "colgroup",
+    "dd", "del", "details", "div", "dl", "dt", "em", "figcaption", "figure", "form",
+    "h1", "h2", "h3", "h4", "head", "hr",
+    "i", "iframe", "image", "img", "input", "ins", "kbd",
+    "label", "li", "nav", "nobr", "ol", "p", "pre", "rgn",
+    "s", "section", "source", "span", "strike", "strong", "sub", "summary", "sup",
+    "table", "tbody", "td", "tfoot", "th", "thead", "tr",
+    "u", "ul", "wbr"
+]
+
+DISALLOWED_HTML_TAG_RULE_LINK = "https://review.learn.microsoft.com/en-us/help/platform/validation-ref/disallowed-html-tag?branch=main"
+
+BROKEN_LINK_RULE_LINK = "https://review.learn.microsoft.com/en-us/help/platform/validation-ref/other-site-link-broken?branch=main"
+
 GLOBAL_EXCLUDE_COMMANDS = ['wait']
 
 EXCLUDE_COMMANDS = {

{azdev-0.1.83 → azdev-0.1.85}/azdev/operations/linter/linter.py

@@ -20,7 +20,7 @@ from azdev.operations.regex import (
     search_argument_context,
     search_command,
     search_command_group)
-from azdev.utilities import diff_branches_detail
+from azdev.utilities import diff_branches_detail, diff_branch_file_patch
 from azdev.utilities.path import get_cli_repo_path, get_ext_repo_paths
 from .util import share_element, exclude_commands, LinterError
 
@@ -63,6 +63,8 @@ class Linter:  # pylint: disable=too-many-public-methods, too-many-instance-attr
         self.git_target = git_target
         self.git_repo = git_repo
         self.exclusions = exclusions
+        self.diffed_lines = set()
+        self._get_diffed_patches()
 
     @property
     def commands(self):
@@ -150,6 +152,17 @@ class Linter:  # pylint: disable=too-many-public-methods, too-many-instance-attr
     def get_parameter_settings(self, command_name, parameter_name):
         return self.get_command_metadata(command_name).arguments.get(parameter_name).type.settings
 
+    def get_parameter_help_info(self, command_name, parameter_name):
+        options = self.get_parameter_options(command_name, parameter_name)
+        command_help = self._loaded_help.get(command_name, None)
+
+        if not command_help:
+            return None
+
+        parameter_helps = command_help.parameters
+        param_help = next((param for param in parameter_helps if share_element(options, param.name.split())), None)
+        return param_help
+
     def command_expired(self, command_name):
         deprecate_info = self._command_loader.command_table[command_name].deprecate_info
         if deprecate_info:
@@ -193,6 +206,10 @@ class Linter:  # pylint: disable=too-many-public-methods, too-many-instance-attr
             return help_entry.short_summary or help_entry.long_summary
         return help_entry
 
+    def get_loaded_help_entry(self, entry):
+        help_entry = self._loaded_help.get(entry, None)
+        return help_entry
+
     def get_command_test_coverage(self):
         diff_index = diff_branches_detail(repo=self.git_repo, target=self.git_target, source=self.git_source)
         commands, _ = self._detect_new_command(diff_index)
@@ -354,6 +371,19 @@ class Linter:  # pylint: disable=too-many-public-methods, too-many-instance-attr
                 'Or add the parameter with missing_parameter_test_coverage rule in linter_exclusions.yml'])
         return exec_state, violations
 
+    def _get_diffed_patches(self):
+        if not self.git_source or not self.git_target or not self.git_repo:
+            return
+        diff_patches = diff_branch_file_patch(repo=self.git_repo, target=self.git_target, source=self.git_source)
+        for change in diff_patches:
+            patch = change.diff.decode("utf-8")
+            added_lines = [line for line in patch.splitlines() if line.startswith('+') and not line.startswith('+++')]
+            self.diffed_lines |= set(added_lines)
+            if added_lines:
+                _logger.info("Changes in file '%s':", change.a_path)
+                for line in added_lines:
+                    _logger.info(line)
+
 
 # pylint: disable=too-many-instance-attributes
 class LinterManager:

azdev-0.1.85/azdev/operations/linter/rules/command_group_rules.py

@@ -0,0 +1,77 @@
+# -----------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# -----------------------------------------------------------------------------
+# pylint: disable=duplicate-code
+
+from azdev.operations.constant import DISALLOWED_HTML_TAG_RULE_LINK
+from ..rule_decorators import CommandGroupRule
+from ..linter import RuleError, LinterSeverity
+from ..util import has_illegal_html_tag, has_broken_site_links
+
+
+@CommandGroupRule(LinterSeverity.HIGH)
+def missing_group_help(linter, command_group_name):
+    if not linter.get_command_group_help(command_group_name) and not linter.command_group_expired(command_group_name) \
+            and command_group_name != '':
+        raise RuleError('Missing help')
+
+
+@CommandGroupRule(LinterSeverity.HIGH)
+def expired_command_group(linter, command_group_name):
+    if linter.command_group_expired(command_group_name):
+        raise RuleError("Deprecated command group is expired and should be removed.")
+
+
+@CommandGroupRule(LinterSeverity.MEDIUM)
+def require_wait_command_if_no_wait(linter, command_group_name):
+    # If any command within a command group or subgroup exposes the --no-wait parameter,
+    # the wait command should be exposed.
+
+    # find commands under this group. A command in this group has one more token than the group name.
+    group_command_names = [cmd for cmd in linter.commands if cmd.startswith(command_group_name) and
+                           len(cmd.split()) == len(command_group_name.split()) + 1]
+
+    # if one of the commands in this group ends with wait we are good
+    for cmd in group_command_names:
+        cmds = cmd.split()
+        if cmds[-1].lower() == "wait":
+            return
+
+    # otherwise there is no wait command. If a command in this group has --no-wait, then error out.
+    for cmd in group_command_names:
+        if linter.get_command_metadata(cmd).supports_no_wait:
+            raise RuleError("Group does not have a 'wait' command, yet '{}' exposes '--no-wait'".format(cmd))
+
+
+@CommandGroupRule(LinterSeverity.MEDIUM)
+def disallowed_html_tag_from_command_group(linter, command_group_name):
+    if command_group_name == '' or not linter.get_loaded_help_entry(command_group_name):
+        return
+    help_entry = linter.get_loaded_help_entry(command_group_name)
+    if help_entry.short_summary and (disallowed_tags := has_illegal_html_tag(help_entry.short_summary,
+                                                                             linter.diffed_lines)):
+        raise RuleError("Disallowed html tags {} in short summary. "
+                        "If the content is a placeholder, please remove <> or wrap it with backtick. "
+                        "For more info please refer to: {}".format(disallowed_tags,
+                                                                   DISALLOWED_HTML_TAG_RULE_LINK))
+    if help_entry.long_summary and (disallowed_tags := has_illegal_html_tag(help_entry.long_summary,
+                                                                            linter.diffed_lines)):
+        raise RuleError("Disallowed html tags {} in long summary. "
+                        "If content is a placeholder, please remove <> or wrap it with backtick. "
+                        "For more info please refer to: {}".format(disallowed_tags,
+                                                                   DISALLOWED_HTML_TAG_RULE_LINK))
+
+
+@CommandGroupRule(LinterSeverity.MEDIUM)
+def broken_site_link_from_command_group(linter, command_group_name):
+    if command_group_name == '' or not linter.get_loaded_help_entry(command_group_name):
+        return
+    help_entry = linter.get_loaded_help_entry(command_group_name)
+    if help_entry.short_summary and (broken_links := has_broken_site_links(help_entry.short_summary)):
+        raise RuleError("Broken links {} in short summary. "
+                        "If link is an example, please wrap it with backtick. ".format(broken_links))
+    if help_entry.long_summary and (broken_links := has_broken_site_links(help_entry.long_summary)):
+        raise RuleError("Broken links {} in long summary. "
+                        "If link is an example, please wrap it with backtick. ".format(broken_links))

azdev-0.1.85/azdev/operations/linter/rules/command_rules.py

@@ -0,0 +1,72 @@
+# -----------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# -----------------------------------------------------------------------------
+# pylint: disable=duplicate-code
+
+from azdev.operations.constant import DISALLOWED_HTML_TAG_RULE_LINK
+from ..rule_decorators import CommandRule
+from ..linter import RuleError, LinterSeverity
+from ..util import has_illegal_html_tag, has_broken_site_links
+
+
+@CommandRule(LinterSeverity.HIGH)
+def missing_command_help(linter, command_name):
+    if not linter.get_command_help(command_name) and not linter.command_expired(command_name):
+        raise RuleError('Missing help')
+
+
+@CommandRule(LinterSeverity.HIGH)
+def no_ids_for_list_commands(linter, command_name):
+    if command_name.split()[-1] == 'list' and 'ids' in linter.get_command_parameters(command_name):
+        raise RuleError('List commands should not expose --ids argument')
+
+
+@CommandRule(LinterSeverity.HIGH)
+def expired_command(linter, command_name):
+    if linter.command_expired(command_name):
+        raise RuleError('Deprecated command is expired and should be removed.')
+
+
+@CommandRule(LinterSeverity.LOW)
+def group_delete_commands_should_confirm(linter, command_name):
+    # We cannot detect from cmd table etc whether a delete command deletes a collection, group or set of resources.
+    # so warn users for every delete command.
+
+    if command_name.split()[-1].lower() == "delete":
+        if 'yes' not in linter.get_command_parameters(command_name):
+            raise RuleError("If this command deletes a collection, or group of resources. "
+                            "Please make sure to ask for confirmation.")
+
+
+@CommandRule(LinterSeverity.MEDIUM)
+def disallowed_html_tag_from_command(linter, command_name):
+    if command_name == '' or not linter.get_loaded_help_entry(command_name):
+        return
+    help_entry = linter.get_loaded_help_entry(command_name)
+    if help_entry.short_summary and (disallowed_tags := has_illegal_html_tag(help_entry.short_summary,
+                                                                             linter.diffed_lines)):
+        raise RuleError("Disallowed html tags {} in short summary. "
+                        "If the content is a placeholder, please remove <> or wrap it with backtick. "
+                        "For more info please refer to: {}".format(disallowed_tags,
+                                                                   DISALLOWED_HTML_TAG_RULE_LINK))
+    if help_entry.long_summary and (disallowed_tags := has_illegal_html_tag(help_entry.long_summary,
+                                                                            linter.diffed_lines)):
+        raise RuleError("Disallowed html tags {} in long summary. "
+                        "If content is a placeholder, please remove <> or wrap it with backtick. "
+                        "For more info please refer to: {}".format(disallowed_tags,
+                                                                   DISALLOWED_HTML_TAG_RULE_LINK))
+
+
+@CommandRule(LinterSeverity.MEDIUM)
+def broken_site_link_from_command(linter, command_name):
+    if command_name == '' or not linter.get_loaded_help_entry(command_name):
+        return
+    help_entry = linter.get_loaded_help_entry(command_name)
+    if help_entry.short_summary and (broken_links := has_broken_site_links(help_entry.short_summary)):
+        raise RuleError("Broken links {} in short summary. "
+                        "If link is an example, please wrap it with backtick. ".format(broken_links))
+    if help_entry.long_summary and (broken_links := has_broken_site_links(help_entry.long_summary)):
+        raise RuleError("Broken links {} in long summary. "
+                        "If link is an example, please wrap it with backtick. ".format(broken_links))

{azdev-0.1.83 → azdev-0.1.85}/azdev/operations/linter/rules/help_rules.py

@@ -20,10 +20,10 @@ from ..util import LinterError
 
 
 # 'az' space then repeating runs of quoted tokens or non quoted characters
-_az_pattern = 'az\s*' + '(([^\"\'])*|' + '((\"[^\"]*\"\s*)|(\'[^\']*\'\s*))' + ')'  # noqa: W605
+_az_pattern = r'az\s*' + '(([^\"\'])*|' + r'((\"[^\"]*\"\s*)|(\'[^\']*\'\s*))' + ')'
 # match the two types of command substitutions
-_CMD_SUB_1 = re.compile("\$\(\s*" + "(" + _az_pattern + ")" + "\)")  # noqa: W605
-_CMD_SUB_2 = re.compile("`\s*" + "(" + _az_pattern + ")" + "`")  # noqa: W605
+_CMD_SUB_1 = re.compile(r"\$\(\s*" + "(" + _az_pattern + ")" + r"\)")
+_CMD_SUB_2 = re.compile(r"`\s*" + "(" + _az_pattern + ")" + "`")
 
 logger = get_logger(__name__)
 

{azdev-0.1.83 → azdev-0.1.85}/azdev/operations/linter/rules/parameter_rules.py

@@ -6,8 +6,10 @@
 
 from knack.deprecation import Deprecated
 
+from azdev.operations.constant import DISALLOWED_HTML_TAG_RULE_LINK
 from ..rule_decorators import ParameterRule
 from ..linter import RuleError, LinterSeverity
+from ..util import has_illegal_html_tag, has_broken_site_links
 
 
 @ParameterRule(LinterSeverity.HIGH)
@@ -170,3 +172,36 @@ def option_should_not_contain_under_score(linter, command_name, parameter_name):
             return
         if '_' in option:
             raise RuleError("Argument's option {} contains '_' which should be '-' instead.".format(option))
+
+
+@ParameterRule(LinterSeverity.MEDIUM)
+def disallowed_html_tag_from_parameter(linter, command_name, parameter_name):
+    if linter.command_expired(command_name) or not linter.get_parameter_help_info(command_name, parameter_name):
+        return
+    help_entry = linter.get_parameter_help_info(command_name, parameter_name)
+    if help_entry.short_summary and (disallowed_tags := has_illegal_html_tag(help_entry.short_summary,
+                                                                             linter.diffed_lines)):
+        raise RuleError("Disallowed html tags {} in short summary. "
+                        "If the content is a placeholder, please remove <> or wrap it with backtick. "
+                        "For more info please refer to: {}".format(disallowed_tags,
+                                                                   DISALLOWED_HTML_TAG_RULE_LINK))
+
+    if help_entry.long_summary and (disallowed_tags := has_illegal_html_tag(help_entry.long_summary,
+                                                                            linter.diffed_lines)):
+        raise RuleError("Disallowed html tags {} in long summary. "
+                        "If content is a placeholder, please remove <> or wrap it with backtick. "
+                        "For more info please refer to: {}".format(disallowed_tags,
+                                                                   DISALLOWED_HTML_TAG_RULE_LINK))
+
+
+@ParameterRule(LinterSeverity.MEDIUM)
+def broken_site_link_from_parameter(linter, command_name, parameter_name):
+    if linter.command_expired(command_name) or not linter.get_parameter_help_info(command_name, parameter_name):
+        return
+    help_entry = linter.get_parameter_help_info(command_name, parameter_name)
+    if help_entry.short_summary and (broken_links := has_broken_site_links(help_entry.short_summary)):
+        raise RuleError("Broken links {} in short summary. "
+                        "If link is an example, please wrap it with backtick. ".format(broken_links))
+    if help_entry.long_summary and (broken_links := has_broken_site_links(help_entry.long_summary)):
+        raise RuleError("Broken links {} in long summary. "
+                        "If link is an example, please wrap it with backtick. ".format(broken_links))

{azdev-0.1.83 → azdev-0.1.85}/azdev/operations/linter/util.py

@@ -6,10 +6,12 @@
 
 import copy
 import re
+import requests
 
 from knack.log import get_logger
 
 from azdev.utilities import get_name_index
+from azdev.operations.constant import ALLOWED_HTML_TAG
 
 
 logger = get_logger(__name__)
@@ -17,6 +19,12 @@ logger = get_logger(__name__)
 
 _LOADER_CLS_RE = re.compile('.*azure/cli/command_modules/(?P<module>[^/]*)/__init__.*')
 
+# add html tag extraction for <abd>, <lun1>, <edge zone>
+# html tag search for <os_des>, <lun1_des> is enabled in cli ci but skipped in doc build cause internal issue
+_HTML_TAG_RE = re.compile(r'<([^\n>]+)>')
+
+_HTTP_LINK_RE = re.compile(r'(?<!`)(https?://[^\s`]+)(?!`)')
+
 
 def filter_modules(command_loader, help_file_entries, modules=None, include_whl_extensions=False):
     """ Modify the command table and help entries to only include certain modules/extensions.
@@ -112,3 +120,38 @@ class LinterError(Exception):
     Exception thrown by linter for non rule violation reasons
     """
     pass  # pylint: disable=unnecessary-pass
+
+
+# pylint: disable=line-too-long
+def has_illegal_html_tag(help_message, filtered_lines=None):
+    """
+    Detect those content wrapped with <> but illegal html tag.
+    Refer to rule doc: https://review.learn.microsoft.com/en-us/help/platform/validation-ref/disallowed-html-tag?branch=main
+    """
+    html_matches = re.findall(_HTML_TAG_RE, help_message)
+    unbackticked_matches = [match for match in html_matches if not re.search(r'`[^`]*' + re.escape('<' + match + '>') + r'[^`]*`', help_message)]
+    disallowed_html_tags = set(unbackticked_matches) - set(ALLOWED_HTML_TAG)
+    if filtered_lines:
+        disallowed_html_tags = [s for s in disallowed_html_tags if any(('<' + s + '>') in diff_line for diff_line in filtered_lines)]
+    return ['<' + s + '>' for s in disallowed_html_tags]
+
+
+def has_broken_site_links(help_message, filtered_lines=None):
+    """
+    Detect broken link in help message.
+    Refer to rule doc: https://review.learn.microsoft.com/en-us/help/platform/validation-ref/other-site-link-broken?branch=main
+    """
+    urls = re.findall(_HTTP_LINK_RE, help_message)
+    invalid_urls = []
+
+    for url in urls:
+        url = re.sub(r'[.")\'\s]*$', '', url)
+        try:
+            response = requests.get(url, timeout=5)
+            if response.status_code != 200:
+                invalid_urls.append(url)
+        except requests.exceptions.RequestException:
+            invalid_urls.append(url)
+    if filtered_lines:
+        invalid_urls = [s for s in invalid_urls if any(s in diff_line for diff_line in filtered_lines)]
+    return invalid_urls

{azdev-0.1.83 → azdev-0.1.85}/azdev/operations/setup.py

@@ -5,6 +5,7 @@
 # -----------------------------------------------------------------------------
 
 import os
+import subprocess
 from shutil import copytree, rmtree
 import time
 
@@ -256,6 +257,111 @@ def _interactive_setup():
         raise CLIError('Installation aborted.')
 
 
+def _setup_azure_cli_repo(cli_path):
+    if cli_path and cli_path != 'EDGE':
+        # Store original directory
+        original_dir = os.getcwd()
+        try:
+            # Change to CLI repo root directory
+            os.chdir(cli_path)
+
+            display(f"\nSetting up Azure CLI repo: {cli_path}\n")
+            # Change git hooks path
+            _change_git_hooks_path(cli_path)
+
+            # Check existing remotes
+            remotes = subprocess.check_output(['git', 'remote', '-v'], text=True)
+
+            # If upstream already exists, nothing to do
+            if 'upstream' in remotes:
+                return
+
+            # Check origin remote URL
+            origin_url = None
+            for line in remotes.splitlines():
+                if line.startswith('origin') and '(fetch)' in line:
+                    origin_url = line.split()[1]
+                    break
+
+            # Only add upstream if origin is an azure-cli fork
+            if origin_url and origin_url.endswith('/azure-cli.git'):
+                upstream_url = 'https://github.com/Azure/azure-cli.git'
+                subprocess.check_call(['git', 'remote', 'add', 'upstream', upstream_url])
+                display(f"Added upstream remote: {upstream_url}")
+                # fetch the upstream/dev branch
+                subprocess.check_call(['git', 'fetch', 'upstream', 'dev'])
+                display(f"Fetched upstream/dev branch for CLI in {cli_path}")
+        except subprocess.CalledProcessError as e:
+            logger.warning("Failed to add upstream remote: %s", str(e))
+        finally:
+            # Always return to original directory
+            os.chdir(original_dir)
+
+
+def _setup_azure_cli_extension_repo(ext_repo_path):
+    if not ext_repo_path:
+        return
+
+    try:
+        # Handle both single path and list of paths
+        repo_paths = ext_repo_path if isinstance(ext_repo_path, list) else [ext_repo_path]
+
+        # Iterate over all repository paths
+        for repo_path in repo_paths:
+            _setup_single_extension_repo(repo_path)
+
+    except subprocess.CalledProcessError as e:
+        logger.warning("Failed to add upstream remote for extensions: %s", str(e))
+
+
+def _setup_single_extension_repo(repo_path):
+    # Store original directory
+    original_dir = os.getcwd()
+
+    try:
+        # Change to extension repo root directory
+        os.chdir(repo_path)
+
+        display(f"\nSetting up Azure CLI extension repo: {repo_path}\n")
+        # Change git hooks path
+        _change_git_hooks_path(repo_path)
+
+        # Check existing remotes
+        remotes = subprocess.check_output(['git', 'remote', '-v'], text=True)
+
+        # If upstream already exists, return
+        if 'upstream' in remotes:
+            return
+
+        # Check origin remote URL
+        origin_url = None
+        for line in remotes.splitlines():
+            if line.startswith('origin') and '(fetch)' in line:
+                origin_url = line.split()[1]
+                break
+
+        # Only add upstream if origin is an azure-cli-extensions fork
+        if origin_url and origin_url.endswith('/azure-cli-extensions.git'):
+            upstream_url = 'https://github.com/Azure/azure-cli-extensions.git'
+            subprocess.check_call(['git', 'remote', 'add', 'upstream', upstream_url])
+            display(f"Added upstream remote for extensions in {repo_path}: {upstream_url}")
+            # fetch the upstream/main branch
+            subprocess.check_call(['git', 'fetch', 'upstream', 'main'])
+            display(f"Fetched upstream/main branch for extensions in {repo_path}")
+
+    finally:
+        # Always return to original directory
+        os.chdir(original_dir)
+
+
+def _change_git_hooks_path(repo_path):
+    # if .githooks folder exists in the repo folder, change the git config to use the .githooks folder in the repo
+    githooks_path = os.path.join(repo_path, '.githooks')
+    if os.path.exists(githooks_path):
+        subprocess.check_call(['git', 'config', 'core.hooksPath', githooks_path])
+        display(f"Changed git hooks path to {githooks_path}")
+
+
 def setup(cli_path=None, ext_repo_path=None, ext=None, deps=None):
 
     require_virtual_env()
@@ -319,6 +425,13 @@ def setup(cli_path=None, ext_repo_path=None, ext=None, deps=None):
     config.set_value('ext', 'repo_paths', dev_sources if dev_sources else '_NONE_')
     config.set_value('cli', 'repo_path', cli_path if cli_path else '_NONE_')
 
+    # Add upstreams for CLI and extensions repos if they are forks
+    if cli_path:
+        _setup_azure_cli_repo(cli_path)
+
+    if ext_repo_path:
+        _setup_azure_cli_extension_repo(ext_repo_path)
+
     # install packages
     subheading('Installing packages')
 

{azdev-0.1.83 → azdev-0.1.85}/azdev/utilities/__init__.py

@@ -35,6 +35,7 @@ from .display import (
 from .git_util import (
     diff_branches,
     filter_by_git_diff,
+    diff_branch_file_patch,
     diff_branches_detail
 )
 from .path import (
@@ -94,5 +95,6 @@ __all__ = [
     'require_virtual_env',
     'require_azure_cli',
     'diff_branches_detail',
+    'diff_branch_file_patch',
     'calc_selected_mod_names',
 ]

{azdev-0.1.83 → azdev-0.1.85}/azdev/utilities/git_util.py

@@ -127,3 +127,39 @@ def diff_branches_detail(repo, target, source):
 
     diff_index = target_commit.diff(source_commit)
     return diff_index
+
+
+def diff_branch_file_patch(repo, target, source):
+    """ Returns compare results of files that have changed in a given repo between two branches.
+        Only focus on these files: _params.py, commands.py, test_*.py """
+    try:
+        import git  # pylint: disable=unused-import,unused-variable
+        import git.exc as git_exc
+        import gitdb
+    except ImportError as ex:
+        raise CLIError(ex)
+
+    from git import Repo
+    try:
+        git_repo = Repo(repo)
+    except (git_exc.NoSuchPathError, git_exc.InvalidGitRepositoryError):
+        raise CLIError('invalid git repo: {}'.format(repo))
+
+    def get_commit(branch):
+        try:
+            return git_repo.commit(branch)
+        except gitdb.exc.BadName:
+            raise CLIError('usage error, invalid branch: {}'.format(branch))
+
+    if source:
+        source_commit = get_commit(source)
+    else:
+        source_commit = git_repo.head.commit
+    target_commit = get_commit(target)
+
+    logger.info('Filtering down to modules which have changed based on:')
+    logger.info('cd %s', repo)
+    logger.info('git --no-pager diff %s..%s --name-only -- .\n', target_commit, source_commit)
+
+    diff_index = target_commit.diff(source_commit, create_patch=True)
+    return diff_index

{azdev-0.1.83 → azdev-0.1.85/azdev.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: azdev
-Version: 0.1.83
+Version: 0.1.85
 Summary: Microsoft Azure CLI Developer Tools
 Home-page: https://github.com/Azure/azure-cli-dev-tools
 Author: Microsoft Corporation
@@ -148,6 +148,15 @@ License
 
 Release History
 ===============
+0.1.85
+++++++
+* `azdev setup`: Setup the upstream and enable .githooks for azure-cli and azure-cli-extensions repos
+* `azdev linter`: Add `disallowed_html_tags` and `broken_site_link` detection in linter rule and set them as `Medium` for unblock CI pipeline temporarily
+
+0.1.84
+++++++
+* `azdev generate-breaking-change-report`: Fix `azdev -h` error caused by global importing `azure.cli.core` in `breaking-change.py` module.
+
 0.1.83
 ++++++
 * `azdev generate-breaking-change-report`: Fix `azdev.operations.breaking_change` not included in `setup.py`.

azdev-0.1.83/azdev/operations/linter/rules/command_group_rules.py

@@ -1,42 +0,0 @@
-# -----------------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for
-# license information.
-# -----------------------------------------------------------------------------
-
-from ..rule_decorators import CommandGroupRule
-from ..linter import RuleError, LinterSeverity
-
-
-@CommandGroupRule(LinterSeverity.HIGH)
-def missing_group_help(linter, command_group_name):
-    if not linter.get_command_group_help(command_group_name) and not linter.command_group_expired(command_group_name) \
-            and command_group_name != '':
-        raise RuleError('Missing help')
-
-
-@CommandGroupRule(LinterSeverity.HIGH)
-def expired_command_group(linter, command_group_name):
-    if linter.command_group_expired(command_group_name):
-        raise RuleError("Deprecated command group is expired and should be removed.")
-
-
-@CommandGroupRule(LinterSeverity.MEDIUM)
-def require_wait_command_if_no_wait(linter, command_group_name):
-    # If any command within a command group or subgroup exposes the --no-wait parameter,
-    # the wait command should be exposed.
-
-    # find commands under this group. A command in this group has one more token than the group name.
-    group_command_names = [cmd for cmd in linter.commands if cmd.startswith(command_group_name) and
-                           len(cmd.split()) == len(command_group_name.split()) + 1]
-
-    # if one of the commands in this group ends with wait we are good
-    for cmd in group_command_names:
-        cmds = cmd.split()
-        if cmds[-1].lower() == "wait":
-            return
-
-    # otherwise there is no wait command. If a command in this group has --no-wait, then error out.
-    for cmd in group_command_names:
-        if linter.get_command_metadata(cmd).supports_no_wait:
-            raise RuleError("Group does not have a 'wait' command, yet '{}' exposes '--no-wait'".format(cmd))

azdev-0.1.83/azdev/operations/linter/rules/command_rules.py

@@ -1,37 +0,0 @@
-# -----------------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for
-# license information.
-# -----------------------------------------------------------------------------
-
-from ..rule_decorators import CommandRule
-from ..linter import RuleError, LinterSeverity
-
-
-@CommandRule(LinterSeverity.HIGH)
-def missing_command_help(linter, command_name):
-    if not linter.get_command_help(command_name) and not linter.command_expired(command_name):
-        raise RuleError('Missing help')
-
-
-@CommandRule(LinterSeverity.HIGH)
-def no_ids_for_list_commands(linter, command_name):
-    if command_name.split()[-1] == 'list' and 'ids' in linter.get_command_parameters(command_name):
-        raise RuleError('List commands should not expose --ids argument')
-
-
-@CommandRule(LinterSeverity.HIGH)
-def expired_command(linter, command_name):
-    if linter.command_expired(command_name):
-        raise RuleError('Deprecated command is expired and should be removed.')
-
-
-@CommandRule(LinterSeverity.LOW)
-def group_delete_commands_should_confirm(linter, command_name):
-    # We cannot detect from cmd table etc whether a delete command deletes a collection, group or set of resources.
-    # so warn users for every delete command.
-
-    if command_name.split()[-1].lower() == "delete":
-        if 'yes' not in linter.get_command_parameters(command_name):
-            raise RuleError("If this command deletes a collection, or group of resources. "
-                            "Please make sure to ask for confirmation.")