azdev 0.1.80__tar.gz → 0.1.81__tar.gz

{azdev-0.1.80 → azdev-0.1.81}/HISTORY.rst

@@ -2,6 +2,10 @@
 
 Release History
 ===============
+0.1.81
+++++++
+* `azdev scan/mask`: Add `--confidence-level` to support secret pattern levels
+
 0.1.80
 ++++++
 * `azdev cmdcov`: Support both extension short name and long name

{azdev-0.1.80/azdev.egg-info → azdev-0.1.81}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: azdev
-Version: 0.1.80
+Version: 0.1.81
 Summary: Microsoft Azure CLI Developer Tools
 Home-page: https://github.com/Azure/azure-cli-dev-tools
 Author: Microsoft Corporation
@@ -40,7 +40,7 @@ Requires-Dist: azure-cli-diff-tool~=0.0.6
 Requires-Dist: packaging
 Requires-Dist: tqdm
 Requires-Dist: wheel==0.30.0
-Requires-Dist: microsoft-security-utilities-secret-masker
+Requires-Dist: microsoft-security-utilities-secret-masker~=1.0.0b2
 
 Microsoft Azure CLI Dev Tools (azdev)
 =====================================
@@ -148,6 +148,10 @@ License
 
 Release History
 ===============
+0.1.81
+++++++
+* `azdev scan/mask`: Add `--confidence-level` to support secret pattern levels
+
 0.1.80
 ++++++
 * `azdev cmdcov`: Support both extension short name and long name

{azdev-0.1.80 → azdev-0.1.81}/azdev/__init__.py

@@ -4,4 +4,4 @@
 # license information.
 # -----------------------------------------------------------------------------
 
-__VERSION__ = '0.1.80'
+__VERSION__ = '0.1.81'

{azdev-0.1.80 → azdev-0.1.81}/azdev/operations/secret.py

@@ -75,19 +75,29 @@ def _get_files_from_directory(directory_path, recursive=None, include_pattern=No
     return target_files
 
 
-def _load_built_in_regex_patterns():
-    return load_regex_patterns_from_json_file('PreciselyClassifiedSecurityKeys.json')
-
-
-def _load_regex_patterns(custom_pattern=None):
-    built_in_regex_patterns = _load_built_in_regex_patterns()
+def _load_built_in_regex_patterns(confidence_level=None):
+    if not confidence_level:
+        confidence_level = 'HIGH'
+    patterns = set()
+    if confidence_level in ['HIGH', 'MEDIUM', 'LOW']:
+        patterns.update(load_regex_patterns_from_json_file('HighConfidenceSecurityModels.json'))
+    if confidence_level in ['MEDIUM', 'LOW']:
+        patterns.update(load_regex_patterns_from_json_file('MediumConfidenceSecurityModels.json'))
+    if confidence_level == 'LOW':
+        patterns.update(load_regex_patterns_from_json_file('LowConfidenceSecurityModels.json'))
+    return patterns
+
+
+def _load_regex_patterns(confidence_level=None, custom_pattern=None):
+    built_in_regex_patterns = _load_built_in_regex_patterns(confidence_level)
 
     if not custom_pattern:
         return built_in_regex_patterns
 
     try:
         if os.path.isfile(custom_pattern):
-            custom_pattern = json.load(custom_pattern)
+            with open(custom_pattern, 'r', encoding='utf8') as f:
+                custom_pattern = json.load(f)
         else:
             custom_pattern = json.loads(custom_pattern)
     except JSONDecodeError as err:
@@ -115,11 +125,11 @@ def _load_regex_patterns(custom_pattern=None):
     return regex_patterns
 
 
-def _scan_secrets_for_string(data, custom_pattern=None):
+def _scan_secrets_for_string(data, confidence_level=None, custom_pattern=None):
     if not data:
         return None
 
-    regex_patterns = _load_regex_patterns(custom_pattern)
+    regex_patterns = _load_regex_patterns(confidence_level, custom_pattern)
     secret_masker = SecretMasker(regex_patterns)
     detected_secrets = secret_masker.detect_secrets(data)
     secrets = []
@@ -135,7 +145,8 @@ def _scan_secrets_for_string(data, custom_pattern=None):
 
 def scan_secrets(file_path=None, directory_path=None, recursive=False,
                  include_pattern=None, exclude_pattern=None, data=None,
-                 save_scan_result=None, scan_result_path=None, custom_pattern=None):
+                 save_scan_result=None, scan_result_path=None,
+                 confidence_level=None, custom_pattern=None):
     _validate_data_path(file_path=file_path, directory_path=directory_path,
                         include_pattern=include_pattern, exclude_pattern=exclude_pattern, data=data)
     target_files = []
@@ -149,7 +160,7 @@ def scan_secrets(file_path=None, directory_path=None, recursive=False,
         target_files.append(file_path)
 
     if data:
-        secrets = _scan_secrets_for_string(data, custom_pattern)
+        secrets = _scan_secrets_for_string(data, confidence_level, custom_pattern)
         if secrets:
             scan_results['raw_data'] = secrets
     elif target_files:
@@ -159,7 +170,7 @@ def scan_secrets(file_path=None, directory_path=None, recursive=False,
                 data = f.read()
             if not data:
                 continue
-            secrets = _scan_secrets_for_string(data, custom_pattern)
+            secrets = _scan_secrets_for_string(data, confidence_level, custom_pattern)
             logger.debug('%d secrets found for %s', len(secrets), target_file)
             if secrets:
                 scan_results[target_file] = secrets
@@ -232,7 +243,8 @@ def _mask_secret_for_string(data, secret, redaction_type=None):
 
 def mask_secrets(file_path=None, directory_path=None, recursive=False,
                  include_pattern=None, exclude_pattern=None, data=None,
-                 save_scan_result=None, scan_result_path=None, custom_pattern=None,
+                 save_scan_result=None, scan_result_path=None,
+                 confidence_level=None, custom_pattern=None,
                  saved_scan_result_path=None, redaction_type='FIXED_VALUE', yes=None):
     scan_results = {}
     if saved_scan_result_path:
@@ -247,7 +259,7 @@ def mask_secrets(file_path=None, directory_path=None, recursive=False,
         scan_response = scan_secrets(file_path=file_path, directory_path=directory_path, recursive=recursive,
                                      include_pattern=include_pattern, exclude_pattern=exclude_pattern, data=data,
                                      save_scan_result=save_scan_result, scan_result_path=scan_result_path,
-                                     custom_pattern=custom_pattern)
+                                     confidence_level=confidence_level, custom_pattern=custom_pattern)
         if save_scan_result and scan_response['scan_result_path']:
             with open(scan_response['scan_result_path'], encoding='utf8') as f:
                 scan_results = json.load(f)

{azdev-0.1.80 → azdev-0.1.81}/azdev/params.py

@@ -125,6 +125,10 @@ def load_arguments(self, _):
                             'If specified, --save-scan-result will be True anyway. '
                             'If not speficied but set --save-scan-result to True, '
                             'the file will be saved as `scan_result_YYYYmmddHHMMSS.json` in your `.azdev` directory ')
+            c.argument('confidence_level', choices=['HIGH', 'MEDIUM', 'LOW'], default='HIGH',
+                       help='Which confidence level can you accept for built-in scanning patterns. If you choose HIGH, '
+                            'we will only scan with high confidence level patterns. If you choose MEDIUM, '
+                            'we will use patterns of medium confidence level or above, which is medium and high level.')
             c.argument('custom_pattern',
                        help='Additional patterns you want to apply or built-in patterns you want to exclude '
                             'for scanning. Can be json string or path to the json file.')

{azdev-0.1.80 → azdev-0.1.81/azdev.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: azdev
-Version: 0.1.80
+Version: 0.1.81
 Summary: Microsoft Azure CLI Developer Tools
 Home-page: https://github.com/Azure/azure-cli-dev-tools
 Author: Microsoft Corporation
@@ -40,7 +40,7 @@ Requires-Dist: azure-cli-diff-tool~=0.0.6
 Requires-Dist: packaging
 Requires-Dist: tqdm
 Requires-Dist: wheel==0.30.0
-Requires-Dist: microsoft-security-utilities-secret-masker
+Requires-Dist: microsoft-security-utilities-secret-masker~=1.0.0b2
 
 Microsoft Azure CLI Dev Tools (azdev)
 =====================================
@@ -148,6 +148,10 @@ License
 
 Release History
 ===============
+0.1.81
+++++++
+* `azdev scan/mask`: Add `--confidence-level` to support secret pattern levels
+
 0.1.80
 ++++++
 * `azdev cmdcov`: Support both extension short name and long name

{azdev-0.1.80 → azdev-0.1.81}/azdev.egg-info/requires.txt

@@ -18,4 +18,4 @@ azure-cli-diff-tool~=0.0.6
 packaging
 tqdm
 wheel==0.30.0
-microsoft-security-utilities-secret-masker
+microsoft-security-utilities-secret-masker~=1.0.0b2

{azdev-0.1.80 → azdev-0.1.81}/setup.py

@@ -86,7 +86,7 @@ setup(
         'packaging',
         'tqdm',
         'wheel==0.30.0',
-        'microsoft-security-utilities-secret-masker'
+        'microsoft-security-utilities-secret-masker~=1.0.0b2'
     ],
     package_data={
         'azdev.config': ['*.*', 'cli_pylintrc', 'ext_pylintrc'],