azdev 0.1.80__tar.gz → 0.1.82__tar.gz

{azdev-0.1.80 → azdev-0.1.82}/HISTORY.rst

@@ -2,6 +2,14 @@
 
 Release History
 ===============
+0.1.82
+++++++
+* `azdev generate-breaking-change-report`: New command to collect upcoming breaking changes from codebase.
+
+0.1.81
+++++++
+* `azdev scan/mask`: Add `--confidence-level` to support secret pattern levels
+
 0.1.80
 ++++++
 * `azdev cmdcov`: Support both extension short name and long name

{azdev-0.1.80/azdev.egg-info → azdev-0.1.82}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: azdev
-Version: 0.1.80
+Version: 0.1.82
 Summary: Microsoft Azure CLI Developer Tools
 Home-page: https://github.com/Azure/azure-cli-dev-tools
 Author: Microsoft Corporation
@@ -40,7 +40,7 @@ Requires-Dist: azure-cli-diff-tool~=0.0.6
 Requires-Dist: packaging
 Requires-Dist: tqdm
 Requires-Dist: wheel==0.30.0
-Requires-Dist: microsoft-security-utilities-secret-masker
+Requires-Dist: microsoft-security-utilities-secret-masker~=1.0.0b2
 
 Microsoft Azure CLI Dev Tools (azdev)
 =====================================
@@ -148,6 +148,14 @@ License
 
 Release History
 ===============
+0.1.82
+++++++
+* `azdev generate-breaking-change-report`: New command to collect upcoming breaking changes from codebase.
+
+0.1.81
+++++++
+* `azdev scan/mask`: Add `--confidence-level` to support secret pattern levels
+
 0.1.80
 ++++++
 * `azdev cmdcov`: Support both extension short name and long name

{azdev-0.1.80 → azdev-0.1.82}/azdev/__init__.py

@@ -4,4 +4,4 @@
 # license information.
 # -----------------------------------------------------------------------------
 
-__VERSION__ = '0.1.80'
+__VERSION__ = '0.1.82'

{azdev-0.1.80 → azdev-0.1.82}/azdev/commands.py

@@ -84,3 +84,6 @@ def load_command_table(self, _):
 
     with CommandGroup(self, 'extension', operation_group('help')) as g:
         g.command('generate-docs', 'generate_extension_ref_docs')
+
+    with CommandGroup(self, '', operation_group('breaking_change')) as g:
+        g.command('generate-breaking-change-report', 'collect_upcoming_breaking_changes')

{azdev-0.1.80 → azdev-0.1.82}/azdev/help.py

@@ -377,3 +377,14 @@ helps['cmdcov'] = """
         - name: Check CLI modules command test coverage in argument level.
           text: azdev cmdcov CLI --level argument
 """
+
+helps['generate-breaking-change-report'] = """
+    short-summary: Collect pre-announced breaking changes items and generate the report.
+    examples:
+        - name: Collect all pre-announced breaking changes, including any that did not specify a target version and group them by target version.
+          text: azdev generate-breaking-change-report CLI --group-by-version --target-version None
+        - name: Collect all pre-announced breaking changes target before next breaking change window, and display them in markdown.
+          text: azdev generate-breaking-change-report CLI --output-format markdown
+        - name: Collect all pre-announced breaking changes in vm, including those failed to specify a target version, and display them in json
+          text: azdev generate-breaking-change-report vm --target-version None
+"""

{azdev-0.1.80 → azdev-0.1.82}/azdev/operations/command_change/__init__.py

@@ -10,7 +10,8 @@ import time
 
 from knack.log import get_logger
 import azure_cli_diff_tool
-from azdev.utilities import display, require_azure_cli, heading, get_path_table, filter_by_git_diff
+from azdev.utilities import display, require_azure_cli, heading, get_path_table, filter_by_git_diff, \
+    calc_selected_mod_names
 from .custom import DiffExportFormat, get_commands_meta, STORED_DEPRECATION_KEY
 from .util import export_commands_meta, dump_command_tree, add_to_command_tree
 from ..statistics import _create_invoker_and_load_cmds, _get_command_source, \
@@ -144,26 +145,7 @@ def cmp_command_meta(base_meta_file, diff_meta_file, only_break=False, output_ty
 def export_command_tree(modules, output_file=None):
     require_azure_cli()
 
-    # allow user to run only on CLI or extensions
-    cli_only = modules == ['CLI']
-    ext_only = modules == ['EXT']
-    if cli_only or ext_only:
-        modules = None
-
-    selected_modules = get_path_table(include_only=modules)
-
-    if cli_only:
-        selected_modules['ext'] = {}
-    if ext_only:
-        selected_modules['core'] = {}
-        selected_modules['mod'] = {}
-
-    if not any(selected_modules.values()):
-        logger.warning('No commands selected to check.')
-
-    selected_mod_names = list(selected_modules['mod'].keys())
-    selected_mod_names += list(selected_modules['ext'].keys())
-    selected_mod_names += list(selected_modules['core'].keys())
+    selected_mod_names = calc_selected_mod_names(modules)
 
     if selected_mod_names:
         display('Modules selected: {}\n'.format(', '.join(selected_mod_names)))

{azdev-0.1.80 → azdev-0.1.82}/azdev/operations/secret.py

@@ -75,19 +75,29 @@ def _get_files_from_directory(directory_path, recursive=None, include_pattern=No
     return target_files
 
 
-def _load_built_in_regex_patterns():
-    return load_regex_patterns_from_json_file('PreciselyClassifiedSecurityKeys.json')
-
-
-def _load_regex_patterns(custom_pattern=None):
-    built_in_regex_patterns = _load_built_in_regex_patterns()
+def _load_built_in_regex_patterns(confidence_level=None):
+    if not confidence_level:
+        confidence_level = 'HIGH'
+    patterns = set()
+    if confidence_level in ['HIGH', 'MEDIUM', 'LOW']:
+        patterns.update(load_regex_patterns_from_json_file('HighConfidenceSecurityModels.json'))
+    if confidence_level in ['MEDIUM', 'LOW']:
+        patterns.update(load_regex_patterns_from_json_file('MediumConfidenceSecurityModels.json'))
+    if confidence_level == 'LOW':
+        patterns.update(load_regex_patterns_from_json_file('LowConfidenceSecurityModels.json'))
+    return patterns
+
+
+def _load_regex_patterns(confidence_level=None, custom_pattern=None):
+    built_in_regex_patterns = _load_built_in_regex_patterns(confidence_level)
 
     if not custom_pattern:
         return built_in_regex_patterns
 
     try:
         if os.path.isfile(custom_pattern):
-            custom_pattern = json.load(custom_pattern)
+            with open(custom_pattern, 'r', encoding='utf8') as f:
+                custom_pattern = json.load(f)
         else:
             custom_pattern = json.loads(custom_pattern)
     except JSONDecodeError as err:
@@ -115,11 +125,11 @@ def _load_regex_patterns(custom_pattern=None):
     return regex_patterns
 
 
-def _scan_secrets_for_string(data, custom_pattern=None):
+def _scan_secrets_for_string(data, confidence_level=None, custom_pattern=None):
     if not data:
         return None
 
-    regex_patterns = _load_regex_patterns(custom_pattern)
+    regex_patterns = _load_regex_patterns(confidence_level, custom_pattern)
     secret_masker = SecretMasker(regex_patterns)
     detected_secrets = secret_masker.detect_secrets(data)
     secrets = []
@@ -135,7 +145,8 @@ def _scan_secrets_for_string(data, custom_pattern=None):
 
 def scan_secrets(file_path=None, directory_path=None, recursive=False,
                  include_pattern=None, exclude_pattern=None, data=None,
-                 save_scan_result=None, scan_result_path=None, custom_pattern=None):
+                 save_scan_result=None, scan_result_path=None,
+                 confidence_level=None, custom_pattern=None):
     _validate_data_path(file_path=file_path, directory_path=directory_path,
                         include_pattern=include_pattern, exclude_pattern=exclude_pattern, data=data)
     target_files = []
@@ -149,7 +160,7 @@ def scan_secrets(file_path=None, directory_path=None, recursive=False,
         target_files.append(file_path)
 
     if data:
-        secrets = _scan_secrets_for_string(data, custom_pattern)
+        secrets = _scan_secrets_for_string(data, confidence_level, custom_pattern)
         if secrets:
             scan_results['raw_data'] = secrets
     elif target_files:
@@ -159,7 +170,7 @@ def scan_secrets(file_path=None, directory_path=None, recursive=False,
                 data = f.read()
             if not data:
                 continue
-            secrets = _scan_secrets_for_string(data, custom_pattern)
+            secrets = _scan_secrets_for_string(data, confidence_level, custom_pattern)
             logger.debug('%d secrets found for %s', len(secrets), target_file)
             if secrets:
                 scan_results[target_file] = secrets
@@ -232,7 +243,8 @@ def _mask_secret_for_string(data, secret, redaction_type=None):
 
 def mask_secrets(file_path=None, directory_path=None, recursive=False,
                  include_pattern=None, exclude_pattern=None, data=None,
-                 save_scan_result=None, scan_result_path=None, custom_pattern=None,
+                 save_scan_result=None, scan_result_path=None,
+                 confidence_level=None, custom_pattern=None,
                  saved_scan_result_path=None, redaction_type='FIXED_VALUE', yes=None):
     scan_results = {}
     if saved_scan_result_path:
@@ -247,7 +259,7 @@ def mask_secrets(file_path=None, directory_path=None, recursive=False,
         scan_response = scan_secrets(file_path=file_path, directory_path=directory_path, recursive=recursive,
                                      include_pattern=include_pattern, exclude_pattern=exclude_pattern, data=data,
                                      save_scan_result=save_scan_result, scan_result_path=scan_result_path,
-                                     custom_pattern=custom_pattern)
+                                     confidence_level=confidence_level, custom_pattern=custom_pattern)
         if save_scan_result and scan_response['scan_result_path']:
             with open(scan_response['scan_result_path'], encoding='utf8') as f:
                 scan_results = json.load(f)

{azdev-0.1.80 → azdev-0.1.82}/azdev/operations/statistics/__init__.py

@@ -194,7 +194,7 @@ def _get_command_source(command_name, command):
     }
 
 
-def _create_invoker_and_load_cmds(cli_ctx):
+def _create_invoker_and_load_cmds(cli_ctx, load_arguments=False):
     from knack.events import (
         EVENT_INVOKER_PRE_CMD_TBL_CREATE, EVENT_INVOKER_POST_CMD_TBL_CREATE)
     from azure.cli.core.commands import register_cache_arguments
@@ -215,9 +215,11 @@ def _create_invoker_and_load_cmds(cli_ctx):
     invoker.commands_loader.load_command_table(None)
     invoker.commands_loader.command_name = ''
 
-    # cli_ctx.raise_event(EVENT_INVOKER_PRE_LOAD_ARGUMENTS, commands_loader=invoker.commands_loader)
-    # invoker.commands_loader.load_arguments()
-    # cli_ctx.raise_event(EVENT_INVOKER_POST_LOAD_ARGUMENTS, commands_loader=invoker.commands_loader)
+    if load_arguments:
+        from azure.cli.core.commands.events import EVENT_INVOKER_PRE_LOAD_ARGUMENTS, EVENT_INVOKER_POST_LOAD_ARGUMENTS
+        cli_ctx.raise_event(EVENT_INVOKER_PRE_LOAD_ARGUMENTS, commands_loader=invoker.commands_loader)
+        invoker.commands_loader.load_arguments()
+        cli_ctx.raise_event(EVENT_INVOKER_POST_LOAD_ARGUMENTS, commands_loader=invoker.commands_loader)
 
     cli_ctx.raise_event(EVENT_INVOKER_POST_CMD_TBL_CREATE, commands_loader=invoker.commands_loader)
     invoker.parser.cli_ctx = cli_ctx

{azdev-0.1.80 → azdev-0.1.82}/azdev/params.py

@@ -125,6 +125,10 @@ def load_arguments(self, _):
                             'If specified, --save-scan-result will be True anyway. '
                             'If not speficied but set --save-scan-result to True, '
                             'the file will be saved as `scan_result_YYYYmmddHHMMSS.json` in your `.azdev` directory ')
+            c.argument('confidence_level', choices=['HIGH', 'MEDIUM', 'LOW'], default='HIGH',
+                       help='Which confidence level can you accept for built-in scanning patterns. If you choose HIGH, '
+                            'we will only scan with high confidence level patterns. If you choose MEDIUM, '
+                            'we will use patterns of medium confidence level or above, which is medium and high level.')
             c.argument('custom_pattern',
                        help='Additional patterns you want to apply or built-in patterns you want to exclude '
                             'for scanning. Can be json string or path to the json file.')
@@ -252,3 +256,17 @@ def load_arguments(self, _):
                                           'If the base directory does not exist, it will be created')
             c.argument('output_type', choices=['xml', 'html', 'text', 'man', 'latex'], default="xml",
                        help='Output type of the generated docs.')
+
+    with ArgumentsContext(self, 'generate-breaking-change-report') as c:
+        c.positional('modules', modules_type)
+        c.argument('target_version', default='NextWindow',
+                   help='Only the breaking changes scheduled prior to the specified version will be displayed. '
+                        'The value could be `NextWindow`, `None` or a specified version like `3.0.0`')
+        c.argument('source', choices=['deprecate_info', 'pre_announce'], default='pre_announce',
+                   help='The source of pre-announced breaking changes. `deprecate_info` represents all breaking changes '
+                        'marked through `deprecation_info`; `pre_announce` represents the breaking changes announced in '
+                        '`breaking_change.py` file.')
+        c.argument('group_by_version', action='store_true',
+                   help='If specified, breaking changes would be grouped by their target version as well.')
+        c.argument('output_format', choices=['structure', 'markdown'], default='structure',
+                   help='Output format of the collected breaking changes.')

{azdev-0.1.80 → azdev-0.1.82}/azdev/utilities/__init__.py

@@ -47,7 +47,8 @@ from .path import (
     get_cli_repo_path,
     get_ext_repo_paths,
     get_path_table,
-    get_name_index
+    get_name_index,
+    calc_selected_mod_names
 )
 from .testing import test_cmd
 from .tools import (
@@ -93,4 +94,5 @@ __all__ = [
     'require_virtual_env',
     'require_azure_cli',
     'diff_branches_detail',
+    'calc_selected_mod_names',
 ]

{azdev-0.1.80 → azdev-0.1.82}/azdev/utilities/path.py

@@ -3,7 +3,7 @@
 # Licensed under the MIT License. See License.txt in the project root for
 # license information.
 # -----------------------------------------------------------------------------
-
+import logging
 import os
 from glob import glob
 
@@ -11,6 +11,8 @@ from knack.util import CLIError
 
 from .const import COMMAND_MODULE_PREFIX, EXTENSION_PREFIX, ENV_VAR_VIRTUAL_ENV
 
+logger = logging.getLogger(__name__)
+
 
 def extract_module_name(path):
 
@@ -261,3 +263,27 @@ def get_path_table(include_only=None, include_whl_extensions=False):
         raise CLIError('unrecognized modules: [ {} ]'.format(', '.join(include_only)))
 
     return table
+
+
+def calc_selected_mod_names(modules=None):
+    # allow user to run only on CLI or extensions
+    cli_only = modules == ['CLI']
+    ext_only = modules == ['EXT']
+    if cli_only or ext_only:
+        modules = None
+
+    selected_modules = get_path_table(include_only=modules)
+
+    if cli_only:
+        selected_modules['ext'] = {}
+    if ext_only:
+        selected_modules['core'] = {}
+        selected_modules['mod'] = {}
+
+    if not any(selected_modules.values()):
+        logger.warning('No commands selected to check.')
+
+    selected_mod_names = list(selected_modules['mod'].keys())
+    selected_mod_names += list(selected_modules['ext'].keys())
+    selected_mod_names += list(selected_modules['core'].keys())
+    return selected_mod_names

{azdev-0.1.80 → azdev-0.1.82/azdev.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: azdev
-Version: 0.1.80
+Version: 0.1.82
 Summary: Microsoft Azure CLI Developer Tools
 Home-page: https://github.com/Azure/azure-cli-dev-tools
 Author: Microsoft Corporation
@@ -40,7 +40,7 @@ Requires-Dist: azure-cli-diff-tool~=0.0.6
 Requires-Dist: packaging
 Requires-Dist: tqdm
 Requires-Dist: wheel==0.30.0
-Requires-Dist: microsoft-security-utilities-secret-masker
+Requires-Dist: microsoft-security-utilities-secret-masker~=1.0.0b2
 
 Microsoft Azure CLI Dev Tools (azdev)
 =====================================
@@ -148,6 +148,14 @@ License
 
 Release History
 ===============
+0.1.82
+++++++
+* `azdev generate-breaking-change-report`: New command to collect upcoming breaking changes from codebase.
+
+0.1.81
+++++++
+* `azdev scan/mask`: Add `--confidence-level` to support secret pattern levels
+
 0.1.80
 ++++++
 * `azdev cmdcov`: Support both extension short name and long name

{azdev-0.1.80 → azdev-0.1.82}/azdev.egg-info/requires.txt

@@ -18,4 +18,4 @@ azure-cli-diff-tool~=0.0.6
 packaging
 tqdm
 wheel==0.30.0
-microsoft-security-utilities-secret-masker
+microsoft-security-utilities-secret-masker~=1.0.0b2

{azdev-0.1.80 → azdev-0.1.82}/setup.py

@@ -86,7 +86,7 @@ setup(
         'packaging',
         'tqdm',
         'wheel==0.30.0',
-        'microsoft-security-utilities-secret-masker'
+        'microsoft-security-utilities-secret-masker~=1.0.0b2'
     ],
     package_data={
         'azdev.config': ['*.*', 'cli_pylintrc', 'ext_pylintrc'],