credsweeper 1.11.5__py3-none-any.whl → 1.12.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of credsweeper might be problematic. Click here for more details.

Files changed (141) hide show
  1. credsweeper/__init__.py +21 -15
  2. credsweeper/__main__.py +141 -35
  3. credsweeper/app.py +11 -11
  4. credsweeper/common/keyword_pattern.py +13 -15
  5. credsweeper/common/morpheme_checklist.txt +2 -0
  6. credsweeper/config/__init__.py +0 -1
  7. credsweeper/config/config.py +3 -3
  8. credsweeper/credentials/__init__.py +0 -5
  9. credsweeper/credentials/augment_candidates.py +1 -1
  10. credsweeper/credentials/candidate.py +1 -1
  11. credsweeper/credentials/credential_manager.py +1 -1
  12. credsweeper/credentials/line_data.py +22 -2
  13. credsweeper/deep_scanner/__init__.py +0 -1
  14. credsweeper/deep_scanner/abstract_scanner.py +3 -3
  15. credsweeper/deep_scanner/byte_scanner.py +1 -1
  16. credsweeper/deep_scanner/bzip2_scanner.py +2 -2
  17. credsweeper/deep_scanner/deb_scanner.py +1 -1
  18. credsweeper/deep_scanner/deep_scanner.py +3 -3
  19. credsweeper/deep_scanner/docx_scanner.py +1 -1
  20. credsweeper/deep_scanner/eml_scanner.py +1 -1
  21. credsweeper/deep_scanner/encoder_scanner.py +1 -1
  22. credsweeper/deep_scanner/gzip_scanner.py +2 -2
  23. credsweeper/deep_scanner/html_scanner.py +1 -1
  24. credsweeper/deep_scanner/jclass_scanner.py +1 -1
  25. credsweeper/deep_scanner/jks_scanner.py +1 -1
  26. credsweeper/deep_scanner/lang_scanner.py +1 -1
  27. credsweeper/deep_scanner/lzma_scanner.py +2 -2
  28. credsweeper/deep_scanner/mxfile_scanner.py +1 -1
  29. credsweeper/deep_scanner/pdf_scanner.py +1 -1
  30. credsweeper/deep_scanner/pkcs_scanner.py +2 -2
  31. credsweeper/deep_scanner/pptx_scanner.py +1 -1
  32. credsweeper/deep_scanner/rpm_scanner.py +1 -1
  33. credsweeper/deep_scanner/tar_scanner.py +2 -2
  34. credsweeper/deep_scanner/tmx_scanner.py +2 -2
  35. credsweeper/deep_scanner/xlsx_scanner.py +2 -2
  36. credsweeper/deep_scanner/xml_scanner.py +1 -1
  37. credsweeper/deep_scanner/zip_scanner.py +2 -2
  38. credsweeper/file_handler/__init__.py +0 -15
  39. credsweeper/file_handler/abstract_provider.py +3 -4
  40. credsweeper/file_handler/byte_content_provider.py +1 -1
  41. credsweeper/file_handler/content_provider.py +1 -1
  42. credsweeper/file_handler/data_content_provider.py +1 -1
  43. credsweeper/file_handler/diff_content_provider.py +133 -3
  44. credsweeper/file_handler/file_path_extractor.py +2 -2
  45. credsweeper/file_handler/files_provider.py +4 -4
  46. credsweeper/file_handler/patches_provider.py +7 -8
  47. credsweeper/file_handler/text_content_provider.py +1 -1
  48. credsweeper/filters/__init__.py +2 -3
  49. credsweeper/filters/filter.py +5 -3
  50. credsweeper/filters/group/__init__.py +0 -2
  51. credsweeper/filters/group/general_keyword.py +2 -2
  52. credsweeper/filters/group/general_pattern.py +2 -2
  53. credsweeper/filters/group/group.py +38 -36
  54. credsweeper/filters/group/password_keyword.py +9 -8
  55. credsweeper/filters/group/token_pattern.py +3 -3
  56. credsweeper/filters/group/url_credentials_group.py +8 -8
  57. credsweeper/filters/group/weird_base36_token.py +3 -3
  58. credsweeper/filters/group/weird_base64_token.py +3 -3
  59. credsweeper/filters/line_git_binary_check.py +5 -4
  60. credsweeper/filters/line_specific_key_check.py +6 -5
  61. credsweeper/filters/line_uue_part_check.py +5 -4
  62. credsweeper/filters/value_allowlist_check.py +6 -5
  63. credsweeper/filters/value_array_dictionary_check.py +8 -6
  64. credsweeper/filters/value_atlassian_token_check.py +6 -5
  65. credsweeper/filters/value_azure_token_check.py +6 -5
  66. credsweeper/filters/value_base32_data_check.py +8 -5
  67. credsweeper/filters/value_base64_data_check.py +6 -5
  68. credsweeper/filters/value_base64_encoded_pem_check.py +6 -5
  69. credsweeper/filters/value_base64_key_check.py +6 -5
  70. credsweeper/filters/value_base64_part_check.py +6 -5
  71. credsweeper/filters/value_basic_auth_check.py +37 -0
  72. credsweeper/filters/value_blocklist_check.py +6 -4
  73. credsweeper/filters/value_camel_case_check.py +6 -5
  74. credsweeper/filters/value_couple_keyword_check.py +6 -4
  75. credsweeper/filters/value_dictionary_keyword_check.py +6 -4
  76. credsweeper/filters/value_discord_bot_check.py +6 -5
  77. credsweeper/filters/value_entropy_base_check.py +6 -5
  78. credsweeper/filters/value_file_path_check.py +8 -5
  79. credsweeper/filters/value_github_check.py +5 -4
  80. credsweeper/filters/value_grafana_check.py +6 -5
  81. credsweeper/filters/value_grafana_service_check.py +5 -4
  82. credsweeper/filters/value_hex_number_check.py +5 -4
  83. credsweeper/filters/value_jfrog_token_check.py +6 -5
  84. credsweeper/filters/value_json_web_key_check.py +6 -5
  85. credsweeper/filters/value_json_web_token_check.py +6 -5
  86. credsweeper/filters/value_last_word_check.py +6 -4
  87. credsweeper/filters/{value_dictionary_value_length_check.py → value_length_check.py} +12 -6
  88. credsweeper/filters/value_method_check.py +5 -4
  89. credsweeper/filters/value_not_allowed_pattern_check.py +6 -5
  90. credsweeper/filters/value_not_part_encoded_check.py +4 -4
  91. credsweeper/filters/value_number_check.py +5 -4
  92. credsweeper/filters/value_pattern_check.py +61 -41
  93. credsweeper/filters/value_similarity_check.py +6 -4
  94. credsweeper/filters/value_split_keyword_check.py +5 -4
  95. credsweeper/filters/value_string_type_check.py +9 -7
  96. credsweeper/filters/value_token_base_check.py +5 -4
  97. credsweeper/filters/value_token_check.py +6 -5
  98. credsweeper/logger/__init__.py +0 -1
  99. credsweeper/logger/logger.py +1 -1
  100. credsweeper/ml_model/__init__.py +0 -1
  101. credsweeper/ml_model/features/__init__.py +1 -0
  102. credsweeper/ml_model/features/entropy_evaluation.py +1 -1
  103. credsweeper/ml_model/features/feature.py +1 -1
  104. credsweeper/ml_model/features/file_extension.py +1 -1
  105. credsweeper/ml_model/features/has_html_tag.py +2 -2
  106. credsweeper/ml_model/features/is_secret_numeric.py +1 -1
  107. credsweeper/ml_model/features/length_of_attribute.py +1 -1
  108. credsweeper/ml_model/features/morpheme_dense.py +15 -8
  109. credsweeper/ml_model/features/rule_name.py +1 -1
  110. credsweeper/ml_model/features/rule_severity.py +21 -0
  111. credsweeper/ml_model/features/search_in_attribute.py +1 -1
  112. credsweeper/ml_model/features/word_in.py +1 -1
  113. credsweeper/ml_model/features/word_in_path.py +5 -2
  114. credsweeper/ml_model/features/word_in_postamble.py +1 -1
  115. credsweeper/ml_model/features/word_in_preamble.py +1 -1
  116. credsweeper/ml_model/features/word_in_transition.py +1 -1
  117. credsweeper/ml_model/features/word_in_value.py +1 -1
  118. credsweeper/ml_model/features/word_in_variable.py +1 -1
  119. credsweeper/ml_model/ml_config.json +130 -24
  120. credsweeper/ml_model/ml_model.onnx +0 -0
  121. credsweeper/ml_model/ml_validator.py +3 -2
  122. credsweeper/rules/__init__.py +0 -1
  123. credsweeper/rules/config.yaml +116 -42
  124. credsweeper/rules/rule.py +4 -3
  125. credsweeper/scanner/__init__.py +0 -1
  126. credsweeper/scanner/scan_type/__init__.py +0 -5
  127. credsweeper/scanner/scan_type/multi_pattern.py +4 -4
  128. credsweeper/scanner/scan_type/pem_key_pattern.py +4 -4
  129. credsweeper/scanner/scan_type/scan_type.py +4 -4
  130. credsweeper/scanner/scan_type/single_pattern.py +4 -4
  131. credsweeper/scanner/scanner.py +12 -8
  132. credsweeper/secret/config.json +2 -2
  133. credsweeper/utils/__init__.py +0 -1
  134. credsweeper/utils/pem_key_detector.py +3 -3
  135. credsweeper/utils/util.py +3 -132
  136. {credsweeper-1.11.5.dist-info → credsweeper-1.12.0.dist-info}/METADATA +1 -1
  137. credsweeper-1.12.0.dist-info/RECORD +161 -0
  138. credsweeper-1.11.5.dist-info/RECORD +0 -159
  139. {credsweeper-1.11.5.dist-info → credsweeper-1.12.0.dist-info}/WHEEL +0 -0
  140. {credsweeper-1.11.5.dist-info → credsweeper-1.12.0.dist-info}/entry_points.txt +0 -0
  141. {credsweeper-1.11.5.dist-info → credsweeper-1.12.0.dist-info}/licenses/LICENSE +0 -0
credsweeper/ml_model/ml_config.json CHANGED
@@ -8,6 +8,10 @@
8
8
  "highest": 0.92996
9
9
  },
10
10
  "features": [
11
+ {
12
+ "type": "RuleSeverity",
13
+ "kwargs": {}
14
+ },
11
15
  {
12
16
  "type": "EntropyEvaluation",
13
17
  "kwargs": {}
@@ -70,6 +74,14 @@
70
74
  "attribute": "value"
71
75
  }
72
76
  },
77
+ {
78
+ "type": "SearchInAttribute",
79
+ "comment": "ASN1 prefix for PEM keys",
80
+ "kwargs": {
81
+ "pattern": "MII",
82
+ "attribute": "value"
83
+ }
84
+ },
73
85
  {
74
86
  "type": "SearchInAttribute",
75
87
  "comment": "camelStyle naming detection",
@@ -122,7 +134,7 @@
122
134
  "type": "SearchInAttribute",
123
135
  "comment": "VariableNotAllowedNameCheck",
124
136
  "kwargs": {
125
- "pattern": "(?i:(id|size|name|type|manager)$)",
137
+ "pattern": "(?i:(id|size|name|type|manager|algorithm|view|error)$)",
126
138
  "attribute": "variable"
127
139
  }
128
140
  },
@@ -139,16 +151,30 @@
139
151
  "kwargs": {
140
152
  "words": [
141
153
  " ",
154
+ ".",
155
+ ",",
156
+ "]",
157
+ "#",
142
158
  "/",
143
- "_at",
159
+ "\\",
144
160
  "_id",
161
+ "_at",
145
162
  "_len",
163
+ "256",
164
+ "512",
146
165
  "access",
166
+ "assert",
147
167
  "cache",
148
168
  "client",
149
169
  "control",
150
- "encrypted",
170
+ "crypt",
171
+ "crypted",
172
+ "decrypt",
173
+ "encrypt",
174
+ "dummy",
175
+ "disable",
151
176
  "example",
177
+ "expect",
152
178
  "expire",
153
179
  "fake",
154
180
  "file",
@@ -173,6 +199,7 @@
173
199
  "pw",
174
200
  "secret",
175
201
  "size",
202
+ "sha",
176
203
  "space",
177
204
  "status",
178
205
  "sword",
@@ -195,6 +222,8 @@
195
222
  "kwargs": {
196
223
  "words": [
197
224
  " ",
225
+ ":",
226
+ "=",
198
227
  "$(",
199
228
  "${",
200
229
  "(",
@@ -207,25 +236,37 @@
207
236
  "[",
208
237
  "_id",
209
238
  "abc",
239
+ "aaaa",
210
240
  "allow",
241
+ "arn:aws:",
211
242
  "bar",
212
243
  "disable",
213
244
  "changeme",
245
+ "crypt",
246
+ "crypted",
247
+ "decrypt",
248
+ "encrypt",
214
249
  "example",
215
250
  "fake",
216
251
  "file",
217
252
  "foo",
253
+ "hash",
254
+ "hex",
255
+ "key",
218
256
  "min",
219
257
  "mock",
220
258
  "my",
221
259
  "nil",
222
260
  "pass",
223
- "passwd",
224
- "password",
225
- "pswd",
226
261
  "public",
227
262
  "pwd",
263
+ "rsa-",
264
+ "salt",
265
+ "secret",
266
+ "sha",
267
+ "ssh",
228
268
  "test",
269
+ "word",
229
270
  "xxx",
230
271
  "xyz"
231
272
  ]
@@ -248,9 +289,12 @@
248
289
  "@",
249
290
  "[",
250
291
  "approval",
292
+ "arn:aws:",
251
293
  "assert",
252
294
  "case",
253
295
  "circle",
296
+ "color",
297
+ "e.g.",
254
298
  "equal",
255
299
  "example",
256
300
  "expect",
@@ -263,7 +307,6 @@
263
307
  "none",
264
308
  "null",
265
309
  "pass",
266
- "password",
267
310
  "path",
268
311
  "pwd",
269
312
  "sqa",
@@ -271,7 +314,8 @@
271
314
  "true",
272
315
  "undefined",
273
316
  "unit",
274
- "width"
317
+ "width",
318
+ "word"
275
319
  ]
276
320
  }
277
321
  },
@@ -284,11 +328,16 @@
284
328
  "&",
285
329
  "(",
286
330
  "->",
331
+ "=>",
287
332
  ".",
333
+ ",",
288
334
  "?",
289
335
  "@",
290
336
  "[",
337
+ "{",
291
338
  "bearer",
339
+ "get",
340
+ "e.g.",
292
341
  "equal",
293
342
  "example",
294
343
  "expect",
@@ -318,9 +367,13 @@
318
367
  "?",
319
368
  "@",
320
369
  "[",
370
+ "}",
371
+ "\\",
321
372
  "assert",
322
373
  "case",
323
374
  "circle",
375
+ "color",
376
+ "e.g.",
324
377
  "equal",
325
378
  "example",
326
379
  "expect",
@@ -333,8 +386,6 @@
333
386
  "none",
334
387
  "null",
335
388
  "pass",
336
- "passwd",
337
- "password",
338
389
  "path",
339
390
  "pwd",
340
391
  "sqa",
@@ -342,7 +393,8 @@
342
393
  "true",
343
394
  "undefined",
344
395
  "unit",
345
- "width"
396
+ "width",
397
+ "word"
346
398
  ]
347
399
  }
348
400
  },
@@ -350,17 +402,62 @@
350
402
  "type": "WordInPath",
351
403
  "kwargs": {
352
404
  "words": [
353
- "/conf",
354
- "/dist-packages/",
355
- "/example",
405
+ "test",
406
+ "mock",
407
+ "/src",
408
+ "code",
409
+ "/include",
410
+ "internal",
411
+ "tool",
412
+ "util",
413
+ "example",
414
+ "sample",
415
+ "conf",
416
+ "secret",
417
+ "setting",
418
+ "security",
419
+ "secure",
420
+ "resource",
421
+ "fixture",
422
+ "docker",
423
+ "/docs",
424
+ "/doc/",
425
+ "document",
426
+ "/lang",
427
+ "/local",
428
+ "/lib",
429
+ "/spec",
430
+ "/pkg",
431
+ "/api",
432
+ "/rest",
433
+ "/opt",
434
+ "/sys",
435
+ "kube",
436
+ "kafka",
437
+ "cluster",
438
+ "template",
439
+ "other",
440
+ "public",
441
+ "init",
442
+ "client",
443
+ "server",
444
+ "/model",
445
+ "/modul",
446
+ "browser",
447
+ "/env/",
448
+ "/app",
449
+ "/assets/",
450
+ "vendor",
451
+ "readme",
452
+ "build",
453
+ "/dist-packages",
356
454
  "/record",
357
455
  "/script",
358
- "/site-packages/",
359
- "/src/",
360
- "/test",
361
- "/tool",
362
- "/usr/local/lib/python",
363
- "/assets/"
456
+ "/site-packages",
457
+ "python",
458
+ "/usr",
459
+ "/etc",
460
+ "/fuzz"
364
461
  ]
365
462
  }
366
463
  },
@@ -378,6 +475,7 @@
378
475
  "kwargs": {
379
476
  "extensions": [
380
477
  "",
478
+ ".04",
381
479
  ".1",
382
480
  ".adoc",
383
481
  ".asciidoc",
@@ -386,17 +484,16 @@
386
484
  ".bat",
387
485
  ".bats",
388
486
  ".bazel",
389
- ".bin",
390
487
  ".build",
391
488
  ".bundle",
392
489
  ".bzl",
393
490
  ".c",
491
+ ".cast",
394
492
  ".cc",
395
493
  ".cf",
396
494
  ".cjs",
397
495
  ".cljc",
398
496
  ".cmd",
399
- ".cmm",
400
497
  ".cnf",
401
498
  ".coffee",
402
499
  ".conf",
@@ -409,6 +506,7 @@
409
506
  ".dist",
410
507
  ".doc",
411
508
  ".dockerfile",
509
+ ".edited",
412
510
  ".eex",
413
511
  ".env",
414
512
  ".erb",
@@ -424,7 +522,9 @@
424
522
  ".go",
425
523
  ".golden",
426
524
  ".gradle",
525
+ ".graphql",
427
526
  ".groovy",
527
+ ".gtpl",
428
528
  ".h",
429
529
  ".haml",
430
530
  ".hs",
@@ -456,6 +556,7 @@
456
556
  ".lua",
457
557
  ".m",
458
558
  ".manifest",
559
+ ".markdown",
459
560
  ".markerb",
460
561
  ".md",
461
562
  ".mdx",
@@ -471,6 +572,7 @@
471
572
  ".odd",
472
573
  ".onnx",
473
574
  ".oracle",
575
+ ".original",
474
576
  ".pan",
475
577
  ".patch",
476
578
  ".php",
@@ -488,17 +590,20 @@
488
590
  ".purs",
489
591
  ".pxd",
490
592
  ".py",
593
+ ".pyi",
491
594
  ".pyx",
492
595
  ".r",
493
596
  ".rake",
494
597
  ".rb",
495
598
  ".re",
496
599
  ".red",
600
+ ".response",
497
601
  ".resx",
498
602
  ".rexx",
499
603
  ".rnh",
500
604
  ".rrc",
501
605
  ".rs",
606
+ ".rsa",
502
607
  ".rsp",
503
608
  ".rst",
504
609
  ".rules",
@@ -512,6 +617,7 @@
512
617
  ".storyboard",
513
618
  ".strings",
514
619
  ".sty",
620
+ ".swift",
515
621
  ".t",
516
622
  ".td",
517
623
  ".tdf",
@@ -530,6 +636,7 @@
530
636
  ".tsx",
531
637
  ".txt",
532
638
  ".var",
639
+ ".vsmdi",
533
640
  ".vue",
534
641
  ".xaml",
535
642
  ".xib",
@@ -550,7 +657,6 @@
550
657
  "CMD Password",
551
658
  "CMD Secret",
552
659
  "CMD Token",
553
- "Certificate",
554
660
  "Credential",
555
661
  "Github Old Token",
556
662
  "Key",
@@ -565,4 +671,4 @@
565
671
  }
566
672
  }
567
673
  ]
568
- }
674
+ }
credsweeper/ml_model/ml_model.onnx CHANGED
Binary file
credsweeper/ml_model/ml_validator.py CHANGED
@@ -9,8 +9,9 @@ from onnxruntime import InferenceSession
9
9
 
10
10
  import credsweeper.ml_model.features as features
11
11
  from credsweeper.common.constants import ThresholdPreset, ML_HUNK
12
- from credsweeper.credentials import Candidate, CandidateKey
13
- from credsweeper.utils import Util
12
+ from credsweeper.credentials.candidate import Candidate
13
+ from credsweeper.credentials.candidate_key import CandidateKey
14
+ from credsweeper.utils.util import Util
14
15
 
15
16
  logger = logging.getLogger(__name__)
16
17
 
credsweeper/rules/__init__.py CHANGED
@@ -1 +0,0 @@
1
- from credsweeper.rules.rule import Rule