azdev 0.1.72__tar.gz → 0.1.74__tar.gz

{azdev-0.1.72 → azdev-0.1.74}/HISTORY.rst

@@ -2,6 +2,15 @@
 
 Release History
 ===============
+0.1.74
+++++++
+* `azdev scan/mask`: New commands for scanning and masking secrets for files or string
+
+0.1.73
+++++++
+* `azdev command-change meta-export`: Add `has_completer` to denote whether completer is configed in arg
+* `azdev command-change meta-export`: Extracting arg help and example for loaded HelpFiles
+
 0.1.72
 ++++++
 * Bump `pylint` to 3

{azdev-0.1.72/azdev.egg-info → azdev-0.1.74}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: azdev
-Version: 0.1.72
+Version: 0.1.74
 Summary: Microsoft Azure CLI Developer Tools
 Home-page: https://github.com/Azure/azure-cli-dev-tools
 Author: Microsoft Corporation
@@ -40,6 +40,7 @@ Requires-Dist: azure-cli-diff-tool~=0.0.6
 Requires-Dist: packaging
 Requires-Dist: tqdm
 Requires-Dist: wheel==0.30.0
+Requires-Dist: microsoft-security-utilities-secret-masker
 
 Microsoft Azure CLI Dev Tools (azdev)
 =====================================
@@ -145,6 +146,15 @@ License
 
 Release History
 ===============
+0.1.74
+++++++
+* `azdev scan/mask`: New commands for scanning and masking secrets for files or string
+
+0.1.73
+++++++
+* `azdev command-change meta-export`: Add `has_completer` to denote whether completer is configed in arg
+* `azdev command-change meta-export`: Extracting arg help and example for loaded HelpFiles
+
 0.1.72
 ++++++
 * Bump `pylint` to 3

{azdev-0.1.72 → azdev-0.1.74}/azdev/__init__.py

@@ -4,4 +4,4 @@
 # license information.
 # -----------------------------------------------------------------------------
 
-__VERSION__ = '0.1.72'
+__VERSION__ = '0.1.74'

{azdev-0.1.72 → azdev-0.1.74}/azdev/commands.py

@@ -9,6 +9,7 @@ from knack.commands import CommandGroup
 from .transformers import performance_benchmark_data_transformer
 
 
+# pylint: disable=too-many-statements
 def load_command_table(self, _):
 
     def operation_group(name):
@@ -27,6 +28,10 @@ def load_command_table(self, _):
     with CommandGroup(self, '', operation_group('linter')) as g:
         g.command('linter', 'run_linter')
 
+    with CommandGroup(self, '', operation_group('secret')) as g:
+        g.command('scan', 'scan_secrets')
+        g.command('mask', 'mask_secrets')
+
     with CommandGroup(self, 'statistics', operation_group('statistics')) as g:
         g.command('list-command-table', 'list_command_table')
         g.command('diff-command-tables', 'diff_command_tables')

{azdev-0.1.72 → azdev-0.1.74}/azdev/help.py

@@ -5,6 +5,7 @@
 # -----------------------------------------------------------------------------
 
 from knack.help_files import helps
+# pylint: disable=line-too-long, anomalous-backslash-in-string
 
 
 helps[''] = """
@@ -159,6 +160,51 @@ helps['linter'] = """
           text: azdev linter --repo azure-cli --tgt upstream/master --src upstream/dev
 """
 
+helps['scan'] = """
+    short-summary: Scan secrets for files or string
+    long-summary: Check built-in scanning rules at https://github.com/microsoft/security-utilities/blob/main/GeneratedRegexPatterns/PreciselyClassifiedSecurityKeys.json
+    examples:
+        - name: Scan secrets for a single file with custom patterns
+          text: |
+                azdev scan --file-path my_file.yaml --custom-pattern my_pattern.json
+                ("my_pattern.json" contains the following content)
+                {
+                    "Include": [
+                        {
+                            "Pattern": "(?<refine>[\w.%#+-]+)(%40|@)([a-z0-9.-]*.[a-z]{2,})",
+                            "Name": "EmailAddress",
+                            "Signatures": ["%40", "@"]
+                        },
+                        {
+                            "Pattern": "(?<refine>[0-9a-f]{8}-?[0-9a-f]{4}-?[0-9a-f]{4}-?[0-9a-f]{4}-?[0-9a-f]{12})",
+                            "Name": "GUID"
+                        }
+                    ],
+                    "Exclude": [
+                        {
+                            "Id": "SEC101/156",
+                            "Name": "AadClientAppIdentifiableCredentials",
+                        }
+                    ]
+                }
+        - name: Scan secrets for raw string and save results to file
+          text: |
+                azdev scan --data "my string waiting to be scanned" --save-scan-result True
+        - name: Recursively scan secrets for a directory and save results to specific file
+          text: |
+                azdev scan --directory-path /path/to/my/folder --recursive --scan-result-path /path/to/scan_result.json
+"""
+
+helps['mask'] = """
+    short-summary: Mask secrets for files or string
+    long-summary: |
+                Redaction type 'FIXED_VALUE' will mask all secrets with '***'.
+                Redaction type 'FIXED_LENGTH' will mask secrets with several '*'s which will keep the original secret length.
+                Redaction type 'SECRET_NAME' redaction type will mask secrets with their secret name (type).
+                Redaction type 'CUSTOM' will mask secrets with 'redaction_token' value you specify through saved scan result file.
+                Check built-in scanning rules at https://github.com/microsoft/security-utilities/blob/main/GeneratedRegexPatterns/PreciselyClassifiedSecurityKeys.json
+"""
+
 helps['statistics'] = """
     short-summary: Commands for CLI modules statistics.
 """

{azdev-0.1.72 → azdev-0.1.74}/azdev/operations/command_change/__init__.py

@@ -4,7 +4,7 @@
 # license information.
 # -----------------------------------------------------------------------------
 
-# pylint: disable=no-else-return, too-many-nested-blocks
+# pylint: disable=no-else-return, too-many-nested-blocks, too-many-locals, too-many-branches
 
 import time
 
@@ -72,6 +72,16 @@ def export_command_meta(modules=None, git_source=None, git_target=None, git_repo
     display('Commands loaded in {} sec'.format(stop - start))
     command_loader = az_cli.invocation.commands_loader
 
+    from azure.cli.core.file_util import get_all_help
+
+    help_info = {}
+    if with_help or with_example:
+        help_files = get_all_help(az_cli)
+        for help_item in help_files:
+            if not help_item.command:
+                continue
+            help_info[help_item.command] = help_item
+
     # trim command table to selected_modules
     command_loader = filter_modules(command_loader, modules=selected_mod_names)
 
@@ -85,7 +95,7 @@ def export_command_meta(modules=None, git_source=None, git_target=None, git_repo
             "name": command_name,
             "source": _get_command_source(command_name, command),
             "is_aaz": False,
-            "help": command.help,
+            "help": help_info[command_name] if command_name in help_info else None,
             "confirmation": command.confirmation is True,
             "arguments": [],
             "az_arguments_schema": None,

{azdev-0.1.72 → azdev-0.1.74}/azdev/operations/command_change/custom.py

@@ -119,6 +119,15 @@ def normalize_para_types(para):
     normalize_para_type(type_bool_opts, "bool")
 
 
+def get_command_examples(command_info, command_meta):
+    example_items = []
+    if command_info and command_info.get("help", None) and hasattr(command_info["help"], "examples"):
+        for example_obj in command_info["help"].examples:
+            example_items.append({"name": example_obj.name, "text": example_obj.text})
+    if example_items:
+        command_meta["examples"] = example_items
+
+
 def gen_command_meta(command_info, with_help=False, with_example=False):
     stored_property_when_exist = ["confirmation", "supports_no_wait", "is_preview", "deprecate_info"]
     command_meta = {
@@ -129,15 +138,10 @@ def gen_command_meta(command_info, with_help=False, with_example=False):
         if command_info.get(prop, None):
             command_meta[prop] = command_info[prop]
     if with_example:
-        try:
-            command_meta["examples"] = command_info["help"]["examples"]
-        except AttributeError:
-            pass
+        get_command_examples(command_info, command_meta)
     if with_help:
-        try:
-            command_meta["desc"] = command_info["help"]["short-summary"]
-        except AttributeError:
-            pass
+        if command_info.get("help", None) and hasattr(command_info["help"], "short_summary"):
+            command_meta["desc"] = command_info["help"].short_summary
     parameters = []
     for _, argument in command_info["arguments"].items():
         if argument.type is None:
@@ -163,13 +167,15 @@ def gen_command_meta(command_info, with_help=False, with_example=False):
             para["id_part"] = settings["id_part"]
         if settings.get("nargs", None):
             para["nargs"] = settings["nargs"]
+        if settings.get("completer", None):
+            para["has_completer"] = True
         if settings.get("default", None):
             if not isinstance(settings["default"], (float, int, str, list, bool)):
                 para["default"] = str(settings["default"])
             else:
                 para["default"] = settings["default"]
         if with_help:
-            para["desc"] = settings["help"]
+            para["desc"] = settings.get("help", "")
         if command_info["is_aaz"] and command_info["az_arguments_schema"]:
             process_aaz_argument(command_info["az_arguments_schema"], settings, para)
         normalize_para_types(para)

azdev-0.1.74/azdev/operations/secret.py

@@ -0,0 +1,247 @@
+# -----------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# -----------------------------------------------------------------------------
+
+import os
+import json
+from json.decoder import JSONDecodeError
+from knack.log import get_logger
+from microsoft_security_utilities_secret_masker import (load_regex_patterns_from_json_file,
+                                                        load_regex_pattern_from_json,
+                                                        SecretMasker)
+logger = get_logger(__name__)
+
+
+def _validate_data_path(file_path=None, directory_path=None, data=None):
+    if file_path and directory_path:
+        raise ValueError('Can not specify file path and directory path at the same time')
+    if file_path and data:
+        raise ValueError('Can not specify file path and raw string at the same time')
+    if directory_path and data:
+        raise ValueError('Can not specify directory path and raw string at the same time')
+    if not file_path and not directory_path and not data:
+        raise ValueError('No file path or directory path or raw string provided')
+
+    if directory_path and not os.path.isdir(directory_path):
+        raise ValueError(f'invalid directory path:{directory_path}')
+    if file_path and not os.path.isfile(file_path):
+        raise ValueError(f'invalid file path:{file_path}')
+
+
+def _load_built_in_regex_patterns():
+    return load_regex_patterns_from_json_file('PreciselyClassifiedSecurityKeys.json')
+
+
+def _load_regex_patterns(custom_pattern=None):
+    built_in_regex_patterns = _load_built_in_regex_patterns()
+
+    if not custom_pattern:
+        return built_in_regex_patterns
+
+    try:
+        if os.path.isfile(custom_pattern):
+            custom_pattern = json.load(custom_pattern)
+        else:
+            custom_pattern = json.loads(custom_pattern)
+    except JSONDecodeError as err:
+        raise ValueError(f'Custom pattern should be in valid json format, err:{err.msg}')
+
+    regex_patterns = []
+    if 'Include' in custom_pattern:
+        for pattern in custom_pattern['Include']:
+            if not pattern.get('Pattern', None):
+                raise ValueError(f'Invalid Custom Pattern: {pattern}, '
+                                 f'"Pattern" property is required for Include patterns')
+            regex_patterns.append(load_regex_pattern_from_json(pattern))
+    if "Exclude" in custom_pattern:
+        exclude_pattern_ids = []
+        for pattern in custom_pattern['Exclude']:
+            if not pattern.get('Id', None):
+                raise ValueError(f'Invalid Custom Pattern: {pattern}, "Id" property is required for Exclude patterns')
+            exclude_pattern_ids.append(pattern['Id'])
+        for pattern in built_in_regex_patterns:
+            if pattern.id in exclude_pattern_ids:
+                continue
+            regex_patterns.append(pattern)
+    else:
+        regex_patterns.extend(built_in_regex_patterns)
+    return regex_patterns
+
+
+def _scan_secrets_for_string(data, custom_pattern=None):
+    if not data:
+        return None
+
+    regex_patterns = _load_regex_patterns(custom_pattern)
+    secret_masker = SecretMasker(regex_patterns)
+    detected_secrets = secret_masker.detect_secrets(data)
+    secrets = []
+    for secret in detected_secrets:
+        secrets.append({
+            'secret_name': secret.name,
+            'secret_value': data[secret.start:secret.end],
+            'secret_index': [secret.start, secret.end],
+            'redaction_token': secret.redaction_token,
+        })
+    return secrets
+
+
+def scan_secrets(file_path=None, directory_path=None, recursive=False, data=None,
+                 save_scan_result=None, scan_result_path=None, custom_pattern=None):
+    _validate_data_path(file_path=file_path, directory_path=directory_path, data=data)
+    target_files = []
+    scan_results = {}
+    if directory_path:
+        directory_path = os.path.abspath(directory_path)
+        if recursive:
+            for root, _, files in os.walk(directory_path):
+                target_files.extend(os.path.join(root, file) for file in files)
+        else:
+            for file in os.listdir(directory_path):
+                file = os.path.join(directory_path, file)
+                if os.path.isfile(file):
+                    target_files.append(file)
+    if file_path:
+        file_path = os.path.abspath(file_path)
+        target_files.append(file_path)
+
+    if data:
+        secrets = _scan_secrets_for_string(data, custom_pattern)
+        if secrets:
+            scan_results['raw_data'] = secrets
+    elif target_files:
+        for target_file in target_files:
+            logger.debug('start scanning secrets for %s', target_file)
+            with open(target_file) as f:
+                data = f.read()
+            if not data:
+                continue
+            secrets = _scan_secrets_for_string(data, custom_pattern)
+            logger.debug('%d secrets found for %s', len(secrets), target_file)
+            if secrets:
+                scan_results[target_file] = secrets
+
+    if scan_result_path:
+        save_scan_result = True
+    if not save_scan_result:
+        return {
+            'secrets_detected': bool(scan_results),
+            'scan_results': scan_results
+        }
+
+    if not scan_results:
+        return {'secrets_detected': False, 'scan_result_path': None}
+
+    if not scan_result_path:
+        from azdev.utilities.config import get_azdev_config_dir
+        from datetime import datetime
+        file_folder = os.path.join(get_azdev_config_dir(), 'scan_results')
+        if not os.path.exists(file_folder):
+            os.mkdir(file_folder, 0o755)
+        file_name = file_path or directory_path or datetime.now().strftime('%Y%m%d%H%M%S')
+        result_file_name = 'scan_result_' + file_name.replace('.', '_') + '.json'
+        scan_result_path = os.path.join(file_folder, result_file_name)
+
+    with open(scan_result_path, 'w') as f:
+        json.dump(scan_results, f)
+        logger.debug('store scanning results in %s', scan_result_path)
+    return {'secrets_detected': True, 'scan_result_path': os.path.abspath(scan_result_path)}
+
+
+def _get_scan_results_from_saved_file(saved_scan_result_path,
+                                      file_path=None, directory_path=None, recursive=False, data=None):
+    scan_results = {}
+    if not os.path.isfile(saved_scan_result_path):
+        raise ValueError(f'invalid saved scan result path:{saved_scan_result_path}')
+    with open(saved_scan_result_path) as f:
+        saved_scan_results = json.load(f)
+    # filter saved scan results to keep those related with specified file(s)
+    _validate_data_path(file_path=file_path, directory_path=directory_path, data=data)
+    if file_path:
+        file_path = os.path.abspath(file_path)
+        if file_path in saved_scan_results:
+            scan_results[file_path] = saved_scan_results[file_path]
+    elif directory_path:
+        if recursive:
+            for root, _, files in os.walk(directory_path):
+                for file in files:
+                    file_full = os.path.join(root, file)
+                    if file_full in saved_scan_results:
+                        scan_results[file_full] = saved_scan_results[file_full]
+        else:
+            for file in os.listdir(directory_path):
+                file_full = os.path.join(directory_path, file)
+                if file_full in saved_scan_results:
+                    scan_results[file_full] = saved_scan_results[file_full]
+    else:
+        scan_results['raw_data'] = saved_scan_results['raw_data']
+
+    return scan_results
+
+
+def _mask_secret_for_string(data, secret, redaction_type=None):
+    if redaction_type == 'FIXED_VALUE':
+        data = data.replace(secret['secret_value'], '***')
+    elif redaction_type == 'FIXED_LENGTH':
+        data = data.replace(secret['secret_value'], '*' * len(secret['secret_value']))
+    elif redaction_type == 'SECRET_NAME':
+        data = data.replace(secret['secret_value'], secret['secret_name'])
+    else:
+        data = data.replace(secret['secret_value'], secret['redaction_token'])
+    return data
+
+
+def mask_secrets(file_path=None, directory_path=None, recursive=False, data=None,
+                 save_scan_result=None, scan_result_path=None, custom_pattern=None,
+                 saved_scan_result_path=None, redaction_type='FIXED_VALUE', yes=None):
+    scan_results = {}
+    if saved_scan_result_path:
+        scan_results = _get_scan_results_from_saved_file(saved_scan_result_path, file_path=file_path,
+                                                         directory_path=directory_path, recursive=recursive, data=data)
+    else:
+        scan_response = scan_secrets(file_path=file_path, directory_path=directory_path, recursive=recursive, data=data,
+                                     save_scan_result=save_scan_result, scan_result_path=scan_result_path,
+                                     custom_pattern=custom_pattern)
+        if save_scan_result and scan_response['scan_result_path']:
+            with open(scan_response['scan_result_path']) as f:
+                scan_results = json.load(f)
+        elif not save_scan_result:
+            scan_results = scan_response['scan_results']
+
+    mask_result = {
+            'mask': False,
+            'data': data,
+            'file_path': file_path,
+            'directory_path': directory_path,
+            'recursive': recursive
+    }
+    if not scan_results:
+        logger.warning('No secrets detected, finish directly.')
+        return mask_result
+    for scan_file_path, secrets in scan_results.items():
+        logger.warning('Will mask %d secrets for %s', len(secrets), scan_file_path)
+    if not yes:
+        from knack.prompting import prompt_y_n
+        if not prompt_y_n(f'Do you want to continue with redaction type {redaction_type}?'):
+            return mask_result
+
+    if 'raw_data' in scan_results:
+        for secret in scan_results['raw_data']:
+            data = _mask_secret_for_string(data, secret, redaction_type)
+        mask_result['mask'] = True
+        mask_result['data'] = data
+        return mask_result
+
+    for scan_file_path, secrets in scan_results.items():
+        with open(scan_file_path, 'r') as f:
+            content = f.read()
+        if not content:
+            continue
+        for secret in secrets:
+            content = _mask_secret_for_string(content, secret, redaction_type)
+        with open(scan_file_path, 'w') as f:
+            f.write(content)
+    mask_result['mask'] = True
+    return mask_result

{azdev-0.1.72 → azdev-0.1.74}/azdev/params.py

@@ -100,6 +100,35 @@ def load_arguments(self, _):
                         'Defaults to "high".')
     # endregion
 
+    # region scan & mask
+    for scope in ['scan', 'mask']:
+        with ArgumentsContext(self, scope) as c:
+            c.argument('file_path', options_list=['--file-path', '-f'],
+                       help='Path of the file you want to scan secrets for')
+            c.argument('directory_path', options_list=['--directory-path', '-d'],
+                       help='Path of the folder you want to scan secrets for')
+            c.argument('recursive', options_list=['--recursive', '-r'],
+                       help='Scan the directory recursively')
+            c.argument('data', help='Raw string you want to scan secrets for')
+            c.argument('save_scan_result', options_list=['--save-scan-result', '--save'], type=bool,
+                       help='Whether to save scan result to file or not')
+            c.argument('scan_result_path', options_list=['--scan-result-path', '--result'],
+                       help='Path for the file you want to save the result in. '
+                            'If specified, --save-scan-result will be True anyway. '
+                            'If not speficied but set --save-scan-result to True, '
+                            'the file will be saved as `scan_result_xxx.json` in your `.azdev` directory ')
+            c.argument('custom_pattern',
+                       help='Additional patterns you want to apply or built-in patterns you want to exclude '
+                            'for scanning. Can be json string or path to the json file.')
+
+    with ArgumentsContext(self, 'mask') as c:
+        c.argument('yes', options_list=['--yes', '-y'], action='store_true', help='Answer "yes" to all prompts.')
+        c.argument('redaction_type', options_list=['--redaction-type', '--type'],
+                   choices=['FIXED_VALUE', 'FIXED_LENGTH', 'SECRET_NAME', 'CUSTOM'])
+        c.argument('saved_scan_result_path', options_list=['--saved-scan-result-path', '--saved-result'],
+                   help='Path of the file you saved the scan result in')
+    # endregion
+
     # region statistics
     with ArgumentsContext(self, 'statistics') as c:
         c.argument('include_whl_extensions',

{azdev-0.1.72 → azdev-0.1.74/azdev.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: azdev
-Version: 0.1.72
+Version: 0.1.74
 Summary: Microsoft Azure CLI Developer Tools
 Home-page: https://github.com/Azure/azure-cli-dev-tools
 Author: Microsoft Corporation
@@ -40,6 +40,7 @@ Requires-Dist: azure-cli-diff-tool~=0.0.6
 Requires-Dist: packaging
 Requires-Dist: tqdm
 Requires-Dist: wheel==0.30.0
+Requires-Dist: microsoft-security-utilities-secret-masker
 
 Microsoft Azure CLI Dev Tools (azdev)
 =====================================
@@ -145,6 +146,15 @@ License
 
 Release History
 ===============
+0.1.74
+++++++
+* `azdev scan/mask`: New commands for scanning and masking secrets for files or string
+
+0.1.73
+++++++
+* `azdev command-change meta-export`: Add `has_completer` to denote whether completer is configed in arg
+* `azdev command-change meta-export`: Extracting arg help and example for loaded HelpFiles
+
 0.1.72
 ++++++
 * Bump `pylint` to 3

{azdev-0.1.72 → azdev-0.1.74}/azdev.egg-info/SOURCES.txt

@@ -48,6 +48,7 @@ azdev/operations/pypi.py
 azdev/operations/python_sdk.py
 azdev/operations/regex.py
 azdev/operations/resource.py
+azdev/operations/secret.py
 azdev/operations/setup.py
 azdev/operations/style.py
 azdev/operations/cmdcov/__init__.py

{azdev-0.1.72 → azdev-0.1.74}/azdev.egg-info/requires.txt

@@ -18,3 +18,4 @@ azure-cli-diff-tool~=0.0.6
 packaging
 tqdm
 wheel==0.30.0
+microsoft-security-utilities-secret-masker

{azdev-0.1.72 → azdev-0.1.74}/setup.py

@@ -85,7 +85,8 @@ setup(
         'azure-cli-diff-tool~=0.0.6',
         'packaging',
         'tqdm',
-        'wheel==0.30.0'
+        'wheel==0.30.0',
+        'microsoft-security-utilities-secret-masker'
     ],
     package_data={
         'azdev.config': ['*.*', 'cli_pylintrc', 'ext_pylintrc'],