eml-parser 2.0.0__tar.gz → 3.0.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {eml_parser-2.0.0 → eml_parser-3.0.6}/.gitignore +3 -1
- {eml_parser-2.0.0/eml_parser.egg-info → eml_parser-3.0.6}/PKG-INFO +15 -26
- {eml_parser-2.0.0 → eml_parser-3.0.6}/README.md +1 -2
- {eml_parser-2.0.0 → eml_parser-3.0.6}/eml_parser/decode.py +7 -3
- {eml_parser-2.0.0 → eml_parser-3.0.6}/eml_parser/parser.py +95 -43
- {eml_parser-2.0.0 → eml_parser-3.0.6}/eml_parser/regexes.py +6 -3
- {eml_parser-2.0.0 → eml_parser-3.0.6}/eml_parser/routing.py +31 -53
- {eml_parser-2.0.0 → eml_parser-3.0.6}/examples/recursively_extract_attachments.py +15 -11
- {eml_parser-2.0.0 → eml_parser-3.0.6}/examples/simple_test.py +12 -22
- eml_parser-3.0.6/pyproject.toml +210 -0
- {eml_parser-2.0.0 → eml_parser-3.0.6}/tests/test_decode.py +10 -0
- {eml_parser-2.0.0 → eml_parser-3.0.6}/tests/test_emlparser.py +61 -13
- eml_parser-3.0.6/tests/test_mimebomb.py +88 -0
- eml_parser-3.0.6/tests/test_pathological_headers.py +39 -0
- {eml_parser-2.0.0 → eml_parser-3.0.6}/tests/test_regexes.py +17 -1
- {eml_parser-2.0.0 → eml_parser-3.0.6}/tests/test_routing.py +5 -0
- eml_parser-2.0.0/.checkignore +0 -5
- eml_parser-2.0.0/.editorconfig +0 -16
- eml_parser-2.0.0/.github/workflows/pypi_publish.yml +0 -27
- eml_parser-2.0.0/.github/workflows/pytest.yml +0 -31
- eml_parser-2.0.0/.jenkinsfile +0 -179
- eml_parser-2.0.0/.readthedocs.yaml +0 -23
- eml_parser-2.0.0/AUTHORS.rst +0 -13
- eml_parser-2.0.0/CHANGELOG.md +0 -179
- eml_parser-2.0.0/PKG-INFO +0 -170
- eml_parser-2.0.0/docs/api.md +0 -3
- eml_parser-2.0.0/docs/index.md +0 -133
- eml_parser-2.0.0/eml_parser.egg-info/SOURCES.txt +0 -57
- eml_parser-2.0.0/eml_parser.egg-info/dependency_links.txt +0 -1
- eml_parser-2.0.0/eml_parser.egg-info/requires.txt +0 -24
- eml_parser-2.0.0/eml_parser.egg-info/top_level.txt +0 -1
- eml_parser-2.0.0/mkdocs.yml +0 -21
- eml_parser-2.0.0/pyproject.toml +0 -179
- eml_parser-2.0.0/samples/github_issue_48.eml +0 -70
- eml_parser-2.0.0/samples/regexes_url_matches.txt +0 -57
- eml_parser-2.0.0/samples/regexes_url_samples.txt +0 -71
- eml_parser-2.0.0/samples/sample.eml +0 -11
- eml_parser-2.0.0/samples/sample_attachments.eml +0 -233
- eml_parser-2.0.0/samples/sample_body_data.eml +0 -26
- eml_parser-2.0.0/samples/sample_body_noscheme_url.eml +0 -26
- eml_parser-2.0.0/samples/sample_bug27257.eml +0 -11
- eml_parser-2.0.0/samples/sample_date_parsing.eml +0 -11
- eml_parser-2.0.0/samples/sample_email_in_from.eml +0 -19
- eml_parser-2.0.0/samples/sample_fail_github_issue_38.eml +0 -207
- eml_parser-2.0.0/samples/sample_gh_issue_76.eml +0 -5
- eml_parser-2.0.0/samples/sample_gh_issue_79_1.eml +0 -5
- eml_parser-2.0.0/samples/sample_gh_issue_79_2.eml +0 -6
- eml_parser-2.0.0/samples/sample_gh_issue_79_3.eml +0 -5
- eml_parser-2.0.0/samples/sample_invalid_date1.eml +0 -9
- eml_parser-2.0.0/samples/sample_large.eml +0 -15680
- eml_parser-2.0.0/samples/sample_mime.eml +0 -665
- eml_parser-2.0.0/samples/sample_mime2.eml +0 -663
- eml_parser-2.0.0/samples/sample_mime_attachment_html.eml +0 -60
- eml_parser-2.0.0/samples/sample_mime_inline_binary.eml +0 -664
- eml_parser-2.0.0/samples/sample_mime_inline_html.eml +0 -59
- eml_parser-2.0.0/setup.cfg +0 -4
- eml_parser-2.0.0/setup.py +0 -5
- eml_parser-2.0.0/tox.ini +0 -25
- {eml_parser-2.0.0 → eml_parser-3.0.6}/LICENSE +0 -0
- {eml_parser-2.0.0 → eml_parser-3.0.6}/eml_parser/__init__.py +0 -0
- {eml_parser-2.0.0 → eml_parser-3.0.6}/eml_parser/py.typed +0 -0
- {eml_parser-2.0.0 → eml_parser-3.0.6}/tests/__init__.py +0 -0
|
@@ -1,49 +1,38 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: eml_parser
|
|
3
|
-
Version:
|
|
3
|
+
Version: 3.0.6
|
|
4
4
|
Summary: Python EML parser library
|
|
5
|
-
Author-email: Georges Toth <georges.toth@govcert.etat.lu>
|
|
6
|
-
License: AGPLv3+
|
|
7
5
|
Project-URL: Homepage, https://github.com/GOVCERT-LU/eml_parser
|
|
8
6
|
Project-URL: Documentation, https://eml-parser.readthedocs.io/
|
|
9
7
|
Project-URL: Source, https://github.com/GOVCERT-LU/eml_parser
|
|
10
8
|
Project-URL: Tracker, https://github.com/GOVCERT-LU/eml_parser/issues
|
|
11
9
|
Project-URL: Changelog, https://github.com/GOVCERT-LU/eml_parser/blob/master/CHANGELOG.md
|
|
10
|
+
Author-email: Georges Toth <georges.toth@govcert.etat.lu>
|
|
11
|
+
License: AGPLv3+
|
|
12
|
+
License-File: LICENSE
|
|
12
13
|
Keywords: email
|
|
13
14
|
Classifier: Development Status :: 5 - Production/Stable
|
|
14
|
-
Classifier: License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)
|
|
15
15
|
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)
|
|
16
17
|
Classifier: Operating System :: OS Independent
|
|
17
18
|
Classifier: Programming Language :: Python :: 3 :: Only
|
|
18
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
19
19
|
Classifier: Programming Language :: Python :: 3.10
|
|
20
20
|
Classifier: Programming Language :: Python :: 3.11
|
|
21
21
|
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
24
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
22
25
|
Classifier: Topic :: Communications :: Email
|
|
23
|
-
Requires-Python: >=3.
|
|
24
|
-
Description-Content-Type: text/markdown
|
|
25
|
-
License-File: LICENSE
|
|
26
|
-
Requires-Dist: python-dateutil
|
|
26
|
+
Requires-Python: >=3.10
|
|
27
27
|
Requires-Dist: charset-normalizer
|
|
28
28
|
Requires-Dist: publicsuffixlist
|
|
29
|
+
Requires-Dist: python-dateutil
|
|
29
30
|
Provides-Extra: filemagic
|
|
30
|
-
Requires-Dist: file-magic>=0.4.0; extra ==
|
|
31
|
+
Requires-Dist: file-magic>=0.4.0; extra == 'filemagic'
|
|
31
32
|
Provides-Extra: regex
|
|
32
|
-
Requires-Dist: regex; extra ==
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
Requires-Dist: mkdocstrings[crystal,python]; extra == "docs"
|
|
36
|
-
Provides-Extra: dev
|
|
37
|
-
Requires-Dist: mypy; extra == "dev"
|
|
38
|
-
Requires-Dist: types-python-dateutil; extra == "dev"
|
|
39
|
-
Requires-Dist: types-chardet; extra == "dev"
|
|
40
|
-
Requires-Dist: ruff; extra == "dev"
|
|
41
|
-
Provides-Extra: test
|
|
42
|
-
Requires-Dist: pytest; extra == "test"
|
|
43
|
-
Requires-Dist: pytest-sugar; extra == "test"
|
|
44
|
-
Requires-Dist: coverage; extra == "test"
|
|
45
|
-
|
|
46
|
-
[](https://codebeat.co/projects/github-com-govcert-lu-eml_parser-master)
|
|
33
|
+
Requires-Dist: regex; extra == 'regex'
|
|
34
|
+
Description-Content-Type: text/markdown
|
|
35
|
+
|
|
47
36
|
[](http://eml-parser.readthedocs.io)
|
|
48
37
|
[](https://badge.fury.io/py/eml-parser)
|
|
49
38
|
[](https://pypi.python.org/pypi/eml-parser/)
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
[](https://codebeat.co/projects/github-com-govcert-lu-eml_parser-master)
|
|
2
1
|
[](http://eml-parser.readthedocs.io)
|
|
3
2
|
[](https://badge.fury.io/py/eml-parser)
|
|
4
3
|
[](https://pypi.python.org/pypi/eml-parser/)
|
|
@@ -122,4 +121,4 @@ Which gives for a minimalistic EML file something like this:
|
|
|
122
121
|
]
|
|
123
122
|
}
|
|
124
123
|
}
|
|
125
|
-
```
|
|
124
|
+
```
|
|
@@ -81,7 +81,7 @@ def decode_field(field: str) -> str:
|
|
|
81
81
|
return string
|
|
82
82
|
|
|
83
83
|
|
|
84
|
-
def decode_string(string: bytes, encoding:
|
|
84
|
+
def decode_string(string: bytes, encoding: str | None = None) -> str:
|
|
85
85
|
"""Try anything possible to parse an encoded bytes string and return the result.
|
|
86
86
|
|
|
87
87
|
We do this using the encoding hint, if this fails, we try to detect the correct
|
|
@@ -214,7 +214,7 @@ def robust_string2date(line: str) -> datetime.datetime:
|
|
|
214
214
|
return date_
|
|
215
215
|
|
|
216
216
|
|
|
217
|
-
def json_serial(obj: typing.Any) ->
|
|
217
|
+
def json_serial(obj: typing.Any) -> str | None:
|
|
218
218
|
"""JSON serializer for objects not serializable by default json code."""
|
|
219
219
|
if isinstance(obj, datetime.datetime):
|
|
220
220
|
if obj.tzinfo is not None:
|
|
@@ -257,7 +257,11 @@ def rfc2047_decode(value: str) -> str:
|
|
|
257
257
|
parsed = ''
|
|
258
258
|
|
|
259
259
|
for k, v in email.header.decode_header(value):
|
|
260
|
-
if
|
|
260
|
+
if isinstance(k, str):
|
|
261
|
+
# decode_header returns str (not bytes) for fragments which did not
|
|
262
|
+
# need RFC2047 decoding, e.g. when the whole value is plain ASCII.
|
|
263
|
+
parsed += k
|
|
264
|
+
elif v is None:
|
|
261
265
|
parsed += k.decode('ascii', errors='ignore')
|
|
262
266
|
|
|
263
267
|
else:
|
|
@@ -105,19 +105,30 @@ class CustomPolicy(email.policy.EmailPolicy):
|
|
|
105
105
|
|
|
106
106
|
return eml_parser.decode.robust_string2date(value).isoformat()
|
|
107
107
|
|
|
108
|
+
elif header in ('sender', 'resent-sender', 'to', 'resent-to', 'cc', 'resent-cc', 'bcc', 'resent-bcc', 'from', 'resent-from', 'reply-to'):
|
|
109
|
+
try:
|
|
110
|
+
return super().header_fetch_parse(name, value)
|
|
111
|
+
except RecursionError:
|
|
112
|
+
# This can happen when the recursion gets too deep in in the stdlib recursive descent parser.
|
|
113
|
+
# In this case, the header is certainly pathological. We still try to extract some addresses.
|
|
114
|
+
m = eml_parser.regexes.email_regex.findall(value)
|
|
115
|
+
return ', '.join(m)
|
|
116
|
+
|
|
108
117
|
return super().header_fetch_parse(name, value)
|
|
109
118
|
|
|
110
119
|
|
|
111
120
|
class EmlParser:
|
|
112
121
|
"""eml-parser class."""
|
|
113
122
|
|
|
123
|
+
MULTIPART_RECURSION_LIMIT = 100
|
|
124
|
+
|
|
114
125
|
# pylint: disable=too-many-arguments
|
|
115
126
|
def __init__(
|
|
116
127
|
self,
|
|
117
128
|
include_raw_body: bool = False,
|
|
118
129
|
include_attachment_data: bool = False,
|
|
119
|
-
pconf:
|
|
120
|
-
policy:
|
|
130
|
+
pconf: dict | None = None,
|
|
131
|
+
policy: email.policy.Policy | None = None,
|
|
121
132
|
ignore_bad_start: bool = False,
|
|
122
133
|
email_force_tld: bool = False,
|
|
123
134
|
domain_force_tld: bool = False,
|
|
@@ -170,6 +181,8 @@ class EmlParser:
|
|
|
170
181
|
|
|
171
182
|
if self.email_force_tld:
|
|
172
183
|
eml_parser.regexes.email_regex = eml_parser.regexes.email_force_tld_regex
|
|
184
|
+
else:
|
|
185
|
+
eml_parser.regexes.email_regex = eml_parser.regexes.email_no_force_tld_regex
|
|
173
186
|
|
|
174
187
|
# If no whitelisting is required, set to emtpy list
|
|
175
188
|
if 'whiteip' not in self.pconf:
|
|
@@ -178,7 +191,7 @@ class EmlParser:
|
|
|
178
191
|
if 'whitefor' not in self.pconf:
|
|
179
192
|
self.pconf['whitefor'] = []
|
|
180
193
|
|
|
181
|
-
self.msg:
|
|
194
|
+
self.msg: email.message.Message | None = None
|
|
182
195
|
|
|
183
196
|
def decode_email(self, eml_file: os.PathLike, ignore_bad_start: bool = False) -> dict:
|
|
184
197
|
"""Function for decoding an EML file into an easily parsable structure.
|
|
@@ -322,7 +335,7 @@ class EmlParser:
|
|
|
322
335
|
# parse and decode Date
|
|
323
336
|
# If date field is present
|
|
324
337
|
if 'date' in self.msg and self.msg.get('date') is not None:
|
|
325
|
-
headers_struc['date'] = datetime.datetime.fromisoformat(typing.cast(str, self.msg.get('date')))
|
|
338
|
+
headers_struc['date'] = datetime.datetime.fromisoformat(typing.cast('str', self.msg.get('date')))
|
|
326
339
|
else:
|
|
327
340
|
# If date field is absent...
|
|
328
341
|
headers_struc['date'] = datetime.datetime.fromisoformat(eml_parser.decode.default_date)
|
|
@@ -400,10 +413,11 @@ class EmlParser:
|
|
|
400
413
|
|
|
401
414
|
# search for e-mail addresses
|
|
402
415
|
for mail_candidate in eml_parser.regexes.email_regex.findall(received_line_flat):
|
|
416
|
+
validated_mail_candidate = mail_candidate
|
|
403
417
|
if self.email_force_tld:
|
|
404
|
-
|
|
405
|
-
if
|
|
406
|
-
headers_struc['received_email'] += [
|
|
418
|
+
validated_mail_candidate = self.get_valid_domain_or_ip(mail_candidate)
|
|
419
|
+
if validated_mail_candidate is not None and validated_mail_candidate not in parsed_routing.get('for', []):
|
|
420
|
+
headers_struc['received_email'] += [validated_mail_candidate]
|
|
407
421
|
|
|
408
422
|
except TypeError: # Ready to parse email without received headers.
|
|
409
423
|
logger.exception('Exception occurred while parsing received lines.')
|
|
@@ -491,6 +505,15 @@ class EmlParser:
|
|
|
491
505
|
if valid_ip:
|
|
492
506
|
list_observed_ip[valid_ip] = 1
|
|
493
507
|
|
|
508
|
+
# URLs do not necessarily appear as-is in the body, as they may contain escaped entities.
|
|
509
|
+
# For this reason, we have to extract the domains again from each parsed URL.
|
|
510
|
+
# Run once over the deduplicated URLs after slicing to keep the cost linear in the body size.
|
|
511
|
+
for url in set(list_observed_urls) | set(list_observed_urls_noscheme):
|
|
512
|
+
for match in eml_parser.regexes.dom_regex.findall(url):
|
|
513
|
+
valid_domain = self.get_valid_domain_or_ip(match.lower())
|
|
514
|
+
if valid_domain:
|
|
515
|
+
list_observed_dom[match.lower()] = 1
|
|
516
|
+
|
|
494
517
|
# Report uri,email and observed domain or hash if no raw body
|
|
495
518
|
if self.include_raw_body:
|
|
496
519
|
if list_observed_urls:
|
|
@@ -536,12 +559,12 @@ class EmlParser:
|
|
|
536
559
|
# "a","titi" ---> c: [truc]
|
|
537
560
|
# "c","truc"
|
|
538
561
|
ch: dict[str, list] = {}
|
|
539
|
-
for
|
|
562
|
+
for header_key, header_value in body_multhead:
|
|
540
563
|
# make sure we are working with strings only
|
|
541
|
-
v = str(
|
|
564
|
+
v = str(header_value)
|
|
542
565
|
|
|
543
566
|
# We are using replace . to : for avoiding issue in mongo
|
|
544
|
-
k =
|
|
567
|
+
k = header_key.lower().replace('.', ':') # Lot of lowers, pre-compute :) .
|
|
545
568
|
|
|
546
569
|
if multipart:
|
|
547
570
|
if k in ch:
|
|
@@ -584,8 +607,8 @@ class EmlParser:
|
|
|
584
607
|
# "a","titi" ---> c: [truc]
|
|
585
608
|
# "c","truc"
|
|
586
609
|
#
|
|
587
|
-
for
|
|
588
|
-
k =
|
|
610
|
+
for header_field_name in set(self.msg.keys()):
|
|
611
|
+
k = header_field_name.lower() # Lot of lower, pre-compute...
|
|
589
612
|
decoded_values = []
|
|
590
613
|
|
|
591
614
|
try:
|
|
@@ -678,7 +701,9 @@ class EmlParser:
|
|
|
678
701
|
else:
|
|
679
702
|
ptr_start = 0
|
|
680
703
|
|
|
681
|
-
for
|
|
704
|
+
for _ptr_end in range(slice_step, body_length + slice_step, slice_step):
|
|
705
|
+
ptr_end = _ptr_end
|
|
706
|
+
|
|
682
707
|
if ' ' in body[ptr_end - 1 : ptr_end]:
|
|
683
708
|
while not (eml_parser.regexes.window_slice_regex.match(body[ptr_end - 1 : ptr_end]) or ptr_end > body_length):
|
|
684
709
|
if ptr_end > body_length:
|
|
@@ -709,7 +734,7 @@ class EmlParser:
|
|
|
709
734
|
|
|
710
735
|
ptr_start = ptr_end
|
|
711
736
|
|
|
712
|
-
def get_valid_domain_or_ip(self, data: str) ->
|
|
737
|
+
def get_valid_domain_or_ip(self, data: str) -> str | None:
|
|
713
738
|
"""Function to determine if an IP address, Email address, or Domain is valid.
|
|
714
739
|
|
|
715
740
|
Args:
|
|
@@ -737,7 +762,7 @@ class EmlParser:
|
|
|
737
762
|
|
|
738
763
|
return None
|
|
739
764
|
|
|
740
|
-
def clean_found_uri(self, url: str) ->
|
|
765
|
+
def clean_found_uri(self, url: str) -> str | None:
|
|
741
766
|
"""Function for validating URLs from the input string.
|
|
742
767
|
|
|
743
768
|
Args:
|
|
@@ -746,6 +771,9 @@ class EmlParser:
|
|
|
746
771
|
Returns:
|
|
747
772
|
str: Returns a valid URL, if found in the input string.
|
|
748
773
|
"""
|
|
774
|
+
if '&' in url:
|
|
775
|
+
url = unescape(url)
|
|
776
|
+
|
|
749
777
|
if '.' not in url and '[' not in url:
|
|
750
778
|
# if we found a URL like e.g. http://afafasasfasfas; that makes no
|
|
751
779
|
# sense, thus skip it, but include http://[2001:db8::1]
|
|
@@ -779,9 +807,6 @@ class EmlParser:
|
|
|
779
807
|
if url.endswith('://'):
|
|
780
808
|
return None
|
|
781
809
|
|
|
782
|
-
if '&' in url:
|
|
783
|
-
url = unescape(url)
|
|
784
|
-
|
|
785
810
|
return url
|
|
786
811
|
|
|
787
812
|
def get_uri_ondata(self, body: str) -> list[str]:
|
|
@@ -836,15 +861,26 @@ class EmlParser:
|
|
|
836
861
|
_field = eml_parser.decode.workaround_field_value_parsing_errors(self.msg, header)
|
|
837
862
|
field = []
|
|
838
863
|
|
|
839
|
-
for
|
|
840
|
-
v = eml_parser.decode.rfc2047_decode(
|
|
864
|
+
for raw_value in _field:
|
|
865
|
+
v = eml_parser.decode.rfc2047_decode(raw_value).replace('\n', '').replace('\r', '')
|
|
841
866
|
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
for
|
|
847
|
-
|
|
867
|
+
try:
|
|
868
|
+
parsing_result: dict[str, typing.Any] = {}
|
|
869
|
+
parser_cls = typing.cast('email.headerregistry.AddressHeader', email.headerregistry.HeaderRegistry()[header])
|
|
870
|
+
parser_cls.parse(v, parsing_result)
|
|
871
|
+
for _group in parsing_result['groups']:
|
|
872
|
+
for _address in _group.addresses:
|
|
873
|
+
field.append((_address.display_name, _address.addr_spec))
|
|
874
|
+
except (AttributeError, IndexError):
|
|
875
|
+
# The stdlib address/header parser can itself raise on certain malformed
|
|
876
|
+
# values (e.g. an unquoted display name ending in a bare period, such as
|
|
877
|
+
# "Test.<test@example.com>"). The exact exception type raised for this
|
|
878
|
+
# case has varied across Python versions, so we defensively catch it here
|
|
879
|
+
# too and fall back to a best-effort regex extraction of e-mail addresses.
|
|
880
|
+
logger.exception('Field value parsing error while parsing "%s", falling back to regex extraction!', v)
|
|
881
|
+
|
|
882
|
+
for m in eml_parser.regexes.email_regex.findall(v):
|
|
883
|
+
field.append(('', m))
|
|
848
884
|
|
|
849
885
|
return_field = []
|
|
850
886
|
|
|
@@ -859,23 +895,31 @@ class EmlParser:
|
|
|
859
895
|
return return_field
|
|
860
896
|
|
|
861
897
|
def get_raw_body_text(
|
|
862
|
-
self, msg: email.message.Message, boundary:
|
|
863
|
-
) -> list[tuple[typing.Any, typing.Any, typing.Any,
|
|
898
|
+
self, msg: email.message.Message, boundary: str | None = None, depth: int = 0
|
|
899
|
+
) -> list[tuple[typing.Any, typing.Any, typing.Any, str | None]]:
|
|
864
900
|
"""This method recursively retrieves all e-mail body parts and returns them as a list.
|
|
865
901
|
|
|
866
902
|
Args:
|
|
867
903
|
msg (email.message.Message): The actual e-mail message or sub-message.
|
|
868
904
|
boundary: Used for passing the boundary marker of multipart messages, and used to easier distinguish different parts.
|
|
905
|
+
depth: Parameter used to track the current recursion level.
|
|
869
906
|
|
|
870
907
|
Returns:
|
|
871
908
|
list: Returns a list of sets which are in the form of "set(encoding, raw_body_string, message field headers, possible boundary marker)"
|
|
909
|
+
|
|
910
|
+
Raises:
|
|
911
|
+
RecursionError: If recursion limit exceeded.
|
|
872
912
|
"""
|
|
873
|
-
raw_body: list[tuple[typing.Any, typing.Any, typing.Any,
|
|
913
|
+
raw_body: list[tuple[typing.Any, typing.Any, typing.Any, str | None]] = []
|
|
914
|
+
|
|
915
|
+
if depth > EmlParser.MULTIPART_RECURSION_LIMIT:
|
|
916
|
+
logger.warning('multi-part nesting limit (%d) exceeded, aborting', EmlParser.MULTIPART_RECURSION_LIMIT)
|
|
917
|
+
raise RecursionError('Recursion limit exceeded')
|
|
874
918
|
|
|
875
919
|
if msg.is_multipart():
|
|
876
920
|
boundary = msg.get_boundary(failobj=None)
|
|
877
921
|
for part in msg.get_payload():
|
|
878
|
-
raw_body.extend(self.get_raw_body_text(typing.cast(email.message.Message, part), boundary=boundary))
|
|
922
|
+
raw_body.extend(self.get_raw_body_text(typing.cast('email.message.Message', part), boundary=boundary, depth=depth + 1))
|
|
879
923
|
else:
|
|
880
924
|
# Treat text document attachments as belonging to the body of the mail.
|
|
881
925
|
# Attachments with a file-extension of .htm/.html are implicitly treated
|
|
@@ -897,14 +941,14 @@ class EmlParser:
|
|
|
897
941
|
|
|
898
942
|
charset = msg.get_content_charset()
|
|
899
943
|
if charset is None:
|
|
900
|
-
raw_body_b = typing.cast(bytes, msg.get_payload(decode=True))
|
|
944
|
+
raw_body_b = typing.cast('bytes', msg.get_payload(decode=True))
|
|
901
945
|
raw_body_str = eml_parser.decode.decode_string(raw_body_b, None)
|
|
902
946
|
else:
|
|
903
947
|
try:
|
|
904
|
-
raw_body_str = typing.cast(bytes, msg.get_payload(decode=True)).decode(charset, 'ignore')
|
|
948
|
+
raw_body_str = typing.cast('bytes', msg.get_payload(decode=True)).decode(charset, 'ignore')
|
|
905
949
|
except (LookupError, ValueError):
|
|
906
950
|
logger.debug('An exception occurred while decoding the payload!', exc_info=True)
|
|
907
|
-
raw_body_str = typing.cast(bytes, msg.get_payload(decode=True)).decode('ascii', 'ignore')
|
|
951
|
+
raw_body_str = typing.cast('bytes', msg.get_payload(decode=True)).decode('ascii', 'ignore')
|
|
908
952
|
|
|
909
953
|
# In case we hit bug 27257 or any other parsing error, try to downgrade the used policy
|
|
910
954
|
try:
|
|
@@ -931,7 +975,7 @@ class EmlParser:
|
|
|
931
975
|
return {k: EmlParser.get_hash(data, k) for k in hashalgo}
|
|
932
976
|
|
|
933
977
|
@staticmethod
|
|
934
|
-
def get_hash(value:
|
|
978
|
+
def get_hash(value: str | bytes, hash_type: str) -> str:
|
|
935
979
|
"""Generate a hash of type *hash_type* for a given value.
|
|
936
980
|
|
|
937
981
|
Args:
|
|
@@ -953,20 +997,28 @@ class EmlParser:
|
|
|
953
997
|
|
|
954
998
|
return hash_algo(_value).hexdigest()
|
|
955
999
|
|
|
956
|
-
def traverse_multipart(self, msg: email.message.Message, counter: int = 0) -> dict[str, typing.Any]:
|
|
1000
|
+
def traverse_multipart(self, msg: email.message.Message, counter: int = 0, depth: int = 0) -> dict[str, typing.Any]:
|
|
957
1001
|
"""Recursively traverses all e-mail message multi-part elements and returns in a parsed form as a dict.
|
|
958
1002
|
|
|
959
1003
|
Args:
|
|
960
1004
|
msg (email.message.Message): An e-mail message object.
|
|
961
1005
|
counter (int, optional): A counter which is used for generating attachments
|
|
962
1006
|
file-names in case there are none found in the header. Default = 0.
|
|
1007
|
+
depth: Parameter used to track the current recursion level.
|
|
963
1008
|
|
|
964
1009
|
Returns:
|
|
965
1010
|
dict: Returns a dict with all original multi-part headers as well as generated hash check-sums,
|
|
966
1011
|
date size, file extension, real mime-type.
|
|
1012
|
+
|
|
1013
|
+
Raises:
|
|
1014
|
+
RecursionError: If recursion limit exceeded.
|
|
967
1015
|
"""
|
|
968
1016
|
attachments = {}
|
|
969
1017
|
|
|
1018
|
+
if depth > EmlParser.MULTIPART_RECURSION_LIMIT:
|
|
1019
|
+
logger.warning('multi-part nesting limit (%d) exceeded, aborting', EmlParser.MULTIPART_RECURSION_LIMIT)
|
|
1020
|
+
raise RecursionError('Recursion limit exceeded')
|
|
1021
|
+
|
|
970
1022
|
if msg.is_multipart():
|
|
971
1023
|
if 'content-type' in msg:
|
|
972
1024
|
if msg.get_content_type() == 'message/rfc822':
|
|
@@ -974,7 +1026,7 @@ class EmlParser:
|
|
|
974
1026
|
attachments.update(self.prepare_multipart_part_attachment(msg, counter))
|
|
975
1027
|
|
|
976
1028
|
for part in msg.get_payload():
|
|
977
|
-
attachments.update(self.traverse_multipart(typing.cast(email.message.EmailMessage, part), counter))
|
|
1029
|
+
attachments.update(self.traverse_multipart(typing.cast('email.message.EmailMessage', part), counter=counter, depth=depth + 1))
|
|
978
1030
|
else:
|
|
979
1031
|
return self.prepare_multipart_part_attachment(msg, counter)
|
|
980
1032
|
|
|
@@ -1013,14 +1065,14 @@ class EmlParser:
|
|
|
1013
1065
|
|
|
1014
1066
|
try:
|
|
1015
1067
|
custom_policy: email.policy.Policy = email.policy.default.clone(max_line_length=0)
|
|
1016
|
-
data = typing.cast(list[email.message.EmailMessage], payload)[0].as_bytes(policy=custom_policy)
|
|
1068
|
+
data = typing.cast('list[email.message.EmailMessage]', payload)[0].as_bytes(policy=custom_policy)
|
|
1017
1069
|
except UnicodeEncodeError:
|
|
1018
1070
|
custom_policy = email.policy.compat32.clone(max_line_length=0)
|
|
1019
|
-
data = typing.cast(list[email.message.EmailMessage], payload)[0].as_bytes(policy=custom_policy)
|
|
1071
|
+
data = typing.cast('list[email.message.EmailMessage]', payload)[0].as_bytes(policy=custom_policy)
|
|
1020
1072
|
|
|
1021
1073
|
file_size = len(data)
|
|
1022
1074
|
else:
|
|
1023
|
-
data = typing.cast(bytes, msg.get_payload(decode=True))
|
|
1075
|
+
data = typing.cast('bytes', msg.get_payload(decode=True))
|
|
1024
1076
|
file_size = len(data)
|
|
1025
1077
|
|
|
1026
1078
|
filename = msg.get_filename('')
|
|
@@ -1055,9 +1107,9 @@ class EmlParser:
|
|
|
1055
1107
|
attachment[file_id]['raw'] = base64.b64encode(data)
|
|
1056
1108
|
|
|
1057
1109
|
ch: dict[str, list[str]] = {}
|
|
1058
|
-
for
|
|
1059
|
-
k =
|
|
1060
|
-
v = str(
|
|
1110
|
+
for header_key, header_value in msg.items():
|
|
1111
|
+
k = header_key.lower()
|
|
1112
|
+
v = str(header_value)
|
|
1061
1113
|
|
|
1062
1114
|
if k in ch:
|
|
1063
1115
|
ch[k].append(v)
|
|
@@ -1071,7 +1123,7 @@ class EmlParser:
|
|
|
1071
1123
|
return attachment
|
|
1072
1124
|
|
|
1073
1125
|
@staticmethod
|
|
1074
|
-
def get_mime_type(data: bytes) ->
|
|
1126
|
+
def get_mime_type(data: bytes) -> tuple[str, str] | tuple[None, None]:
|
|
1075
1127
|
"""Get mime-type information based on the provided bytes object.
|
|
1076
1128
|
|
|
1077
1129
|
Args:
|
|
@@ -14,8 +14,12 @@ __license__ = 'AGPL v3+'
|
|
|
14
14
|
|
|
15
15
|
# regex compilation
|
|
16
16
|
# W3C HTML5 standard recommended regex for e-mail validation
|
|
17
|
-
|
|
18
|
-
|
|
17
|
+
# It also implements the 64 character limit on the local part (RFC 5321, section 2.4), with a safety factor.
|
|
18
|
+
# This mitigates an issue with quadratic complexity when searching for e-mail addresses in very long strings.
|
|
19
|
+
email_no_force_tld_regex = re.compile(r"""([a-zA-Z0-9.!#$%&'*+\-/=?^_`{|}~]{1,128}@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*)""", re.MULTILINE)
|
|
20
|
+
email_force_tld_regex = re.compile(r"""([a-zA-Z0-9.!#$%&'*+\-/=?^_`{|}~]{1,128}@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)+)""", re.MULTILINE)
|
|
21
|
+
email_regex = email_no_force_tld_regex
|
|
22
|
+
|
|
19
23
|
|
|
20
24
|
# regex for detecting RFC2047 encodings - used from https://dmorgan.info/posts/encoded-word-syntax/
|
|
21
25
|
email_regex_rfc2047 = re.compile(r"""=\?{1}([\w\S]+)\?{1}([B|Q|b|q])\?{1}([\w\S]+)\?{1}=""")
|
|
@@ -96,7 +100,6 @@ url_regex_href = re.compile(
|
|
|
96
100
|
)
|
|
97
101
|
|
|
98
102
|
date_regex = re.compile(r""";[ \w\s:,+\-()]+$""")
|
|
99
|
-
noparenthesis_regex = re.compile(r"""\([^()]*\)""")
|
|
100
103
|
cleanline_regex = re.compile(r"""(^[;\s]{0,}|[;\s]{0,}$)""")
|
|
101
104
|
|
|
102
105
|
escape_special_regex_chars = re.compile(r"""([\^$\[\]()+?.])""")
|
|
@@ -8,29 +8,24 @@ import eml_parser.regexes
|
|
|
8
8
|
|
|
9
9
|
|
|
10
10
|
def noparenthesis(line: str) -> str:
|
|
11
|
-
"""Remove nested
|
|
12
|
-
|
|
13
|
-
@FIXME rewrite this function.
|
|
11
|
+
"""Remove nested parentheses, until no pairs of matching parentyheses are present.
|
|
14
12
|
|
|
15
13
|
Args:
|
|
16
|
-
line (str): Input text to search in for
|
|
14
|
+
line (str): Input text to search in for parentheses.
|
|
17
15
|
|
|
18
16
|
Returns:
|
|
19
|
-
str: Return a string with all
|
|
17
|
+
str: Return a string with all text in matching parentheses removed. For example,
|
|
18
|
+
noparenthesis('a((b)c(d') == 'a(c(d'
|
|
20
19
|
"""
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
if lline == line_:
|
|
31
|
-
break
|
|
32
|
-
|
|
33
|
-
return line_
|
|
20
|
+
fragments: list[list[str]] = [[]]
|
|
21
|
+
for ch in line:
|
|
22
|
+
if ch == '(':
|
|
23
|
+
fragments.append([ch])
|
|
24
|
+
else:
|
|
25
|
+
fragments[-1].append(ch)
|
|
26
|
+
if ch == ')' and len(fragments) > 1:
|
|
27
|
+
fragments.pop()
|
|
28
|
+
return ''.join(ch for w in fragments for ch in w)
|
|
34
29
|
|
|
35
30
|
|
|
36
31
|
def cleanline(line: str) -> str:
|
|
@@ -63,7 +58,7 @@ def get_domain_ip(line: str) -> list[str]:
|
|
|
63
58
|
|
|
64
59
|
|
|
65
60
|
def parserouting(line: str) -> dict[str, typing.Any]:
|
|
66
|
-
"""This method tries to
|
|
61
|
+
"""This method tries to parse an e-mail header received line and extract machine-readable information.
|
|
67
62
|
|
|
68
63
|
Note that there are a large number of formats for these lines
|
|
69
64
|
and a lot of weird ones which are not commonly used.
|
|
@@ -105,49 +100,32 @@ def parserouting(line: str) -> dict[str, typing.Any]:
|
|
|
105
100
|
npline = npline.strip(' ') # Remove any border WhiteSpace
|
|
106
101
|
|
|
107
102
|
borders = ['from ', 'by ', 'with ', 'for ']
|
|
108
|
-
result: list[dict[str, typing.Any]] = []
|
|
109
103
|
|
|
110
104
|
# Scan the line to determine the order, and presence of each "from/by/with/for" words
|
|
105
|
+
raw_matches: list[tuple[int, str]] = []
|
|
111
106
|
for word in borders:
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
end = npline.find(endword)
|
|
118
|
-
if end < loc or end == -1:
|
|
119
|
-
end = 0xFFFFFFF # Kindof MAX 31 bits
|
|
120
|
-
result.append({'name_in': word, 'pos': loc, 'name_out': endword, 'weight': end + loc})
|
|
121
|
-
# print({'name_in': word, 'pos': loc, 'name_out': endword, 'weight': end+loc})
|
|
122
|
-
|
|
123
|
-
# Create the word list... "from/by/with/for" by sorting the list.
|
|
124
|
-
if not result:
|
|
107
|
+
pos = npline.find(word)
|
|
108
|
+
if pos != -1:
|
|
109
|
+
raw_matches.append((pos, word))
|
|
110
|
+
|
|
111
|
+
if not raw_matches:
|
|
125
112
|
out['warning'] = ['Nothing Parsable']
|
|
126
113
|
return out
|
|
127
114
|
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
result_max = 0xFFFFFFFF
|
|
131
|
-
line_max: dict[str, typing.Any] = {}
|
|
132
|
-
for eline in result:
|
|
133
|
-
if eline['name_in'] == word and eline['weight'] <= result_max:
|
|
134
|
-
result_max = eline['weight']
|
|
135
|
-
line_max = eline
|
|
115
|
+
# Sort keywords sequentially by their appearance order in the header
|
|
116
|
+
tout = sorted(raw_matches, key=lambda match: match[0])
|
|
136
117
|
|
|
137
|
-
|
|
138
|
-
|
|
118
|
+
reg = ''
|
|
119
|
+
for i, item in enumerate(tout):
|
|
120
|
+
field = item[1].strip()
|
|
121
|
+
kw = item[1]
|
|
139
122
|
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
tout = sorted(tout, key=lambda x: typing.cast(int, x[0]))
|
|
123
|
+
# Capture lazily until the next keyword
|
|
124
|
+
reg += kw + r'(?P<' + field + r'>.*?)'
|
|
143
125
|
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
reg += item[1] + '(?P<' + item[1].strip() + '>.*)' # type: ignore
|
|
148
|
-
if npdate:
|
|
149
|
-
# escape special regex chars
|
|
150
|
-
reg += eml_parser.regexes.escape_special_regex_chars.sub(r"""\\\1""", npdate)
|
|
126
|
+
if i + 1 >= len(tout):
|
|
127
|
+
# Last field captures lazily until a date separator ';' or end of string
|
|
128
|
+
reg += '(?:;|$)'
|
|
151
129
|
|
|
152
130
|
reparse = re.compile(reg)
|
|
153
131
|
reparseg = reparse.search(line)
|