eml-parser 2.0.0__tar.gz → 3.0.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. {eml_parser-2.0.0 → eml_parser-3.0.6}/.gitignore +3 -1
  2. {eml_parser-2.0.0/eml_parser.egg-info → eml_parser-3.0.6}/PKG-INFO +15 -26
  3. {eml_parser-2.0.0 → eml_parser-3.0.6}/README.md +1 -2
  4. {eml_parser-2.0.0 → eml_parser-3.0.6}/eml_parser/decode.py +7 -3
  5. {eml_parser-2.0.0 → eml_parser-3.0.6}/eml_parser/parser.py +95 -43
  6. {eml_parser-2.0.0 → eml_parser-3.0.6}/eml_parser/regexes.py +6 -3
  7. {eml_parser-2.0.0 → eml_parser-3.0.6}/eml_parser/routing.py +31 -53
  8. {eml_parser-2.0.0 → eml_parser-3.0.6}/examples/recursively_extract_attachments.py +15 -11
  9. {eml_parser-2.0.0 → eml_parser-3.0.6}/examples/simple_test.py +12 -22
  10. eml_parser-3.0.6/pyproject.toml +210 -0
  11. {eml_parser-2.0.0 → eml_parser-3.0.6}/tests/test_decode.py +10 -0
  12. {eml_parser-2.0.0 → eml_parser-3.0.6}/tests/test_emlparser.py +61 -13
  13. eml_parser-3.0.6/tests/test_mimebomb.py +88 -0
  14. eml_parser-3.0.6/tests/test_pathological_headers.py +39 -0
  15. {eml_parser-2.0.0 → eml_parser-3.0.6}/tests/test_regexes.py +17 -1
  16. {eml_parser-2.0.0 → eml_parser-3.0.6}/tests/test_routing.py +5 -0
  17. eml_parser-2.0.0/.checkignore +0 -5
  18. eml_parser-2.0.0/.editorconfig +0 -16
  19. eml_parser-2.0.0/.github/workflows/pypi_publish.yml +0 -27
  20. eml_parser-2.0.0/.github/workflows/pytest.yml +0 -31
  21. eml_parser-2.0.0/.jenkinsfile +0 -179
  22. eml_parser-2.0.0/.readthedocs.yaml +0 -23
  23. eml_parser-2.0.0/AUTHORS.rst +0 -13
  24. eml_parser-2.0.0/CHANGELOG.md +0 -179
  25. eml_parser-2.0.0/PKG-INFO +0 -170
  26. eml_parser-2.0.0/docs/api.md +0 -3
  27. eml_parser-2.0.0/docs/index.md +0 -133
  28. eml_parser-2.0.0/eml_parser.egg-info/SOURCES.txt +0 -57
  29. eml_parser-2.0.0/eml_parser.egg-info/dependency_links.txt +0 -1
  30. eml_parser-2.0.0/eml_parser.egg-info/requires.txt +0 -24
  31. eml_parser-2.0.0/eml_parser.egg-info/top_level.txt +0 -1
  32. eml_parser-2.0.0/mkdocs.yml +0 -21
  33. eml_parser-2.0.0/pyproject.toml +0 -179
  34. eml_parser-2.0.0/samples/github_issue_48.eml +0 -70
  35. eml_parser-2.0.0/samples/regexes_url_matches.txt +0 -57
  36. eml_parser-2.0.0/samples/regexes_url_samples.txt +0 -71
  37. eml_parser-2.0.0/samples/sample.eml +0 -11
  38. eml_parser-2.0.0/samples/sample_attachments.eml +0 -233
  39. eml_parser-2.0.0/samples/sample_body_data.eml +0 -26
  40. eml_parser-2.0.0/samples/sample_body_noscheme_url.eml +0 -26
  41. eml_parser-2.0.0/samples/sample_bug27257.eml +0 -11
  42. eml_parser-2.0.0/samples/sample_date_parsing.eml +0 -11
  43. eml_parser-2.0.0/samples/sample_email_in_from.eml +0 -19
  44. eml_parser-2.0.0/samples/sample_fail_github_issue_38.eml +0 -207
  45. eml_parser-2.0.0/samples/sample_gh_issue_76.eml +0 -5
  46. eml_parser-2.0.0/samples/sample_gh_issue_79_1.eml +0 -5
  47. eml_parser-2.0.0/samples/sample_gh_issue_79_2.eml +0 -6
  48. eml_parser-2.0.0/samples/sample_gh_issue_79_3.eml +0 -5
  49. eml_parser-2.0.0/samples/sample_invalid_date1.eml +0 -9
  50. eml_parser-2.0.0/samples/sample_large.eml +0 -15680
  51. eml_parser-2.0.0/samples/sample_mime.eml +0 -665
  52. eml_parser-2.0.0/samples/sample_mime2.eml +0 -663
  53. eml_parser-2.0.0/samples/sample_mime_attachment_html.eml +0 -60
  54. eml_parser-2.0.0/samples/sample_mime_inline_binary.eml +0 -664
  55. eml_parser-2.0.0/samples/sample_mime_inline_html.eml +0 -59
  56. eml_parser-2.0.0/setup.cfg +0 -4
  57. eml_parser-2.0.0/setup.py +0 -5
  58. eml_parser-2.0.0/tox.ini +0 -25
  59. {eml_parser-2.0.0 → eml_parser-3.0.6}/LICENSE +0 -0
  60. {eml_parser-2.0.0 → eml_parser-3.0.6}/eml_parser/__init__.py +0 -0
  61. {eml_parser-2.0.0 → eml_parser-3.0.6}/eml_parser/py.typed +0 -0
  62. {eml_parser-2.0.0 → eml_parser-3.0.6}/tests/__init__.py +0 -0
@@ -1,5 +1,6 @@
1
1
  *.eml
2
2
  *.pyc
3
+ uv.lock
3
4
 
4
5
  build
5
6
  dist
@@ -11,4 +12,5 @@ eml_parser.egg-info
11
12
  .mypy_cache
12
13
  .pytest_cache
13
14
  venv
14
- .vscode
15
+ .venv
16
+ .vscode
@@ -1,49 +1,38 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.5
2
2
  Name: eml_parser
3
- Version: 2.0.0
3
+ Version: 3.0.6
4
4
  Summary: Python EML parser library
5
- Author-email: Georges Toth <georges.toth@govcert.etat.lu>
6
- License: AGPLv3+
7
5
  Project-URL: Homepage, https://github.com/GOVCERT-LU/eml_parser
8
6
  Project-URL: Documentation, https://eml-parser.readthedocs.io/
9
7
  Project-URL: Source, https://github.com/GOVCERT-LU/eml_parser
10
8
  Project-URL: Tracker, https://github.com/GOVCERT-LU/eml_parser/issues
11
9
  Project-URL: Changelog, https://github.com/GOVCERT-LU/eml_parser/blob/master/CHANGELOG.md
10
+ Author-email: Georges Toth <georges.toth@govcert.etat.lu>
11
+ License: AGPLv3+
12
+ License-File: LICENSE
12
13
  Keywords: email
13
14
  Classifier: Development Status :: 5 - Production/Stable
14
- Classifier: License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)
15
15
  Classifier: Intended Audience :: Developers
16
+ Classifier: License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)
16
17
  Classifier: Operating System :: OS Independent
17
18
  Classifier: Programming Language :: Python :: 3 :: Only
18
- Classifier: Programming Language :: Python :: 3.9
19
19
  Classifier: Programming Language :: Python :: 3.10
20
20
  Classifier: Programming Language :: Python :: 3.11
21
21
  Classifier: Programming Language :: Python :: 3.12
22
+ Classifier: Programming Language :: Python :: 3.13
23
+ Classifier: Programming Language :: Python :: 3.14
24
+ Classifier: Programming Language :: Python :: 3.15
22
25
  Classifier: Topic :: Communications :: Email
23
- Requires-Python: >=3.9
24
- Description-Content-Type: text/markdown
25
- License-File: LICENSE
26
- Requires-Dist: python-dateutil
26
+ Requires-Python: >=3.10
27
27
  Requires-Dist: charset-normalizer
28
28
  Requires-Dist: publicsuffixlist
29
+ Requires-Dist: python-dateutil
29
30
  Provides-Extra: filemagic
30
- Requires-Dist: file-magic>=0.4.0; extra == "filemagic"
31
+ Requires-Dist: file-magic>=0.4.0; extra == 'filemagic'
31
32
  Provides-Extra: regex
32
- Requires-Dist: regex; extra == "regex"
33
- Provides-Extra: docs
34
- Requires-Dist: mkdocs-material; extra == "docs"
35
- Requires-Dist: mkdocstrings[crystal,python]; extra == "docs"
36
- Provides-Extra: dev
37
- Requires-Dist: mypy; extra == "dev"
38
- Requires-Dist: types-python-dateutil; extra == "dev"
39
- Requires-Dist: types-chardet; extra == "dev"
40
- Requires-Dist: ruff; extra == "dev"
41
- Provides-Extra: test
42
- Requires-Dist: pytest; extra == "test"
43
- Requires-Dist: pytest-sugar; extra == "test"
44
- Requires-Dist: coverage; extra == "test"
45
-
46
- [![Code Health](https://codebeat.co/badges/d631cfb2-a7f8-476a-9d2e-58e58db95bc8)](https://codebeat.co/projects/github-com-govcert-lu-eml_parser-master)
33
+ Requires-Dist: regex; extra == 'regex'
34
+ Description-Content-Type: text/markdown
35
+
47
36
  [![Documentation Status](https://readthedocs.org/projects/eml-parser/badge/)](http://eml-parser.readthedocs.io)
48
37
  [![PyPI](https://badge.fury.io/py/eml-parser.svg)](https://badge.fury.io/py/eml-parser)
49
38
  [![PyPI pyversions](https://img.shields.io/pypi/pyversions/eml-parser.svg)](https://pypi.python.org/pypi/eml-parser/)
@@ -1,4 +1,3 @@
1
- [![Code Health](https://codebeat.co/badges/d631cfb2-a7f8-476a-9d2e-58e58db95bc8)](https://codebeat.co/projects/github-com-govcert-lu-eml_parser-master)
2
1
  [![Documentation Status](https://readthedocs.org/projects/eml-parser/badge/)](http://eml-parser.readthedocs.io)
3
2
  [![PyPI](https://badge.fury.io/py/eml-parser.svg)](https://badge.fury.io/py/eml-parser)
4
3
  [![PyPI pyversions](https://img.shields.io/pypi/pyversions/eml-parser.svg)](https://pypi.python.org/pypi/eml-parser/)
@@ -122,4 +121,4 @@ Which gives for a minimalistic EML file something like this:
122
121
  ]
123
122
  }
124
123
  }
125
- ```
124
+ ```
@@ -81,7 +81,7 @@ def decode_field(field: str) -> str:
81
81
  return string
82
82
 
83
83
 
84
- def decode_string(string: bytes, encoding: typing.Optional[str] = None) -> str:
84
+ def decode_string(string: bytes, encoding: str | None = None) -> str:
85
85
  """Try anything possible to parse an encoded bytes string and return the result.
86
86
 
87
87
  We do this using the encoding hint, if this fails, we try to detect the correct
@@ -214,7 +214,7 @@ def robust_string2date(line: str) -> datetime.datetime:
214
214
  return date_
215
215
 
216
216
 
217
- def json_serial(obj: typing.Any) -> typing.Optional[str]:
217
+ def json_serial(obj: typing.Any) -> str | None:
218
218
  """JSON serializer for objects not serializable by default json code."""
219
219
  if isinstance(obj, datetime.datetime):
220
220
  if obj.tzinfo is not None:
@@ -257,7 +257,11 @@ def rfc2047_decode(value: str) -> str:
257
257
  parsed = ''
258
258
 
259
259
  for k, v in email.header.decode_header(value):
260
- if v is None:
260
+ if isinstance(k, str):
261
+ # decode_header returns str (not bytes) for fragments which did not
262
+ # need RFC2047 decoding, e.g. when the whole value is plain ASCII.
263
+ parsed += k
264
+ elif v is None:
261
265
  parsed += k.decode('ascii', errors='ignore')
262
266
 
263
267
  else:
@@ -105,19 +105,30 @@ class CustomPolicy(email.policy.EmailPolicy):
105
105
 
106
106
  return eml_parser.decode.robust_string2date(value).isoformat()
107
107
 
108
+ elif header in ('sender', 'resent-sender', 'to', 'resent-to', 'cc', 'resent-cc', 'bcc', 'resent-bcc', 'from', 'resent-from', 'reply-to'):
109
+ try:
110
+ return super().header_fetch_parse(name, value)
111
+ except RecursionError:
112
+ # This can happen when the recursion gets too deep in in the stdlib recursive descent parser.
113
+ # In this case, the header is certainly pathological. We still try to extract some addresses.
114
+ m = eml_parser.regexes.email_regex.findall(value)
115
+ return ', '.join(m)
116
+
108
117
  return super().header_fetch_parse(name, value)
109
118
 
110
119
 
111
120
  class EmlParser:
112
121
  """eml-parser class."""
113
122
 
123
+ MULTIPART_RECURSION_LIMIT = 100
124
+
114
125
  # pylint: disable=too-many-arguments
115
126
  def __init__(
116
127
  self,
117
128
  include_raw_body: bool = False,
118
129
  include_attachment_data: bool = False,
119
- pconf: typing.Optional[dict] = None,
120
- policy: typing.Optional[email.policy.Policy] = None,
130
+ pconf: dict | None = None,
131
+ policy: email.policy.Policy | None = None,
121
132
  ignore_bad_start: bool = False,
122
133
  email_force_tld: bool = False,
123
134
  domain_force_tld: bool = False,
@@ -170,6 +181,8 @@ class EmlParser:
170
181
 
171
182
  if self.email_force_tld:
172
183
  eml_parser.regexes.email_regex = eml_parser.regexes.email_force_tld_regex
184
+ else:
185
+ eml_parser.regexes.email_regex = eml_parser.regexes.email_no_force_tld_regex
173
186
 
174
187
  # If no whitelisting is required, set to emtpy list
175
188
  if 'whiteip' not in self.pconf:
@@ -178,7 +191,7 @@ class EmlParser:
178
191
  if 'whitefor' not in self.pconf:
179
192
  self.pconf['whitefor'] = []
180
193
 
181
- self.msg: typing.Optional[email.message.Message] = None
194
+ self.msg: email.message.Message | None = None
182
195
 
183
196
  def decode_email(self, eml_file: os.PathLike, ignore_bad_start: bool = False) -> dict:
184
197
  """Function for decoding an EML file into an easily parsable structure.
@@ -322,7 +335,7 @@ class EmlParser:
322
335
  # parse and decode Date
323
336
  # If date field is present
324
337
  if 'date' in self.msg and self.msg.get('date') is not None:
325
- headers_struc['date'] = datetime.datetime.fromisoformat(typing.cast(str, self.msg.get('date')))
338
+ headers_struc['date'] = datetime.datetime.fromisoformat(typing.cast('str', self.msg.get('date')))
326
339
  else:
327
340
  # If date field is absent...
328
341
  headers_struc['date'] = datetime.datetime.fromisoformat(eml_parser.decode.default_date)
@@ -400,10 +413,11 @@ class EmlParser:
400
413
 
401
414
  # search for e-mail addresses
402
415
  for mail_candidate in eml_parser.regexes.email_regex.findall(received_line_flat):
416
+ validated_mail_candidate = mail_candidate
403
417
  if self.email_force_tld:
404
- mail_candidate = self.get_valid_domain_or_ip(mail_candidate)
405
- if mail_candidate is not None and mail_candidate not in parsed_routing.get('for', []):
406
- headers_struc['received_email'] += [mail_candidate]
418
+ validated_mail_candidate = self.get_valid_domain_or_ip(mail_candidate)
419
+ if validated_mail_candidate is not None and validated_mail_candidate not in parsed_routing.get('for', []):
420
+ headers_struc['received_email'] += [validated_mail_candidate]
407
421
 
408
422
  except TypeError: # Ready to parse email without received headers.
409
423
  logger.exception('Exception occurred while parsing received lines.')
@@ -491,6 +505,15 @@ class EmlParser:
491
505
  if valid_ip:
492
506
  list_observed_ip[valid_ip] = 1
493
507
 
508
+ # URLs do not necessarily appear as-is in the body, as they may contain escaped entities.
509
+ # For this reason, we have to extract the domains again from each parsed URL.
510
+ # Run once over the deduplicated URLs after slicing to keep the cost linear in the body size.
511
+ for url in set(list_observed_urls) | set(list_observed_urls_noscheme):
512
+ for match in eml_parser.regexes.dom_regex.findall(url):
513
+ valid_domain = self.get_valid_domain_or_ip(match.lower())
514
+ if valid_domain:
515
+ list_observed_dom[match.lower()] = 1
516
+
494
517
  # Report uri,email and observed domain or hash if no raw body
495
518
  if self.include_raw_body:
496
519
  if list_observed_urls:
@@ -536,12 +559,12 @@ class EmlParser:
536
559
  # "a","titi" ---> c: [truc]
537
560
  # "c","truc"
538
561
  ch: dict[str, list] = {}
539
- for k, v in body_multhead:
562
+ for header_key, header_value in body_multhead:
540
563
  # make sure we are working with strings only
541
- v = str(v)
564
+ v = str(header_value)
542
565
 
543
566
  # We are using replace . to : for avoiding issue in mongo
544
- k = k.lower().replace('.', ':') # Lot of lowers, pre-compute :) .
567
+ k = header_key.lower().replace('.', ':') # Lot of lowers, pre-compute :) .
545
568
 
546
569
  if multipart:
547
570
  if k in ch:
@@ -584,8 +607,8 @@ class EmlParser:
584
607
  # "a","titi" ---> c: [truc]
585
608
  # "c","truc"
586
609
  #
587
- for k in set(self.msg.keys()):
588
- k = k.lower() # Lot of lower, pre-compute...
610
+ for header_field_name in set(self.msg.keys()):
611
+ k = header_field_name.lower() # Lot of lower, pre-compute...
589
612
  decoded_values = []
590
613
 
591
614
  try:
@@ -678,7 +701,9 @@ class EmlParser:
678
701
  else:
679
702
  ptr_start = 0
680
703
 
681
- for ptr_end in range(slice_step, body_length + slice_step, slice_step):
704
+ for _ptr_end in range(slice_step, body_length + slice_step, slice_step):
705
+ ptr_end = _ptr_end
706
+
682
707
  if ' ' in body[ptr_end - 1 : ptr_end]:
683
708
  while not (eml_parser.regexes.window_slice_regex.match(body[ptr_end - 1 : ptr_end]) or ptr_end > body_length):
684
709
  if ptr_end > body_length:
@@ -709,7 +734,7 @@ class EmlParser:
709
734
 
710
735
  ptr_start = ptr_end
711
736
 
712
- def get_valid_domain_or_ip(self, data: str) -> typing.Optional[str]:
737
+ def get_valid_domain_or_ip(self, data: str) -> str | None:
713
738
  """Function to determine if an IP address, Email address, or Domain is valid.
714
739
 
715
740
  Args:
@@ -737,7 +762,7 @@ class EmlParser:
737
762
 
738
763
  return None
739
764
 
740
- def clean_found_uri(self, url: str) -> typing.Optional[str]:
765
+ def clean_found_uri(self, url: str) -> str | None:
741
766
  """Function for validating URLs from the input string.
742
767
 
743
768
  Args:
@@ -746,6 +771,9 @@ class EmlParser:
746
771
  Returns:
747
772
  str: Returns a valid URL, if found in the input string.
748
773
  """
774
+ if '&' in url:
775
+ url = unescape(url)
776
+
749
777
  if '.' not in url and '[' not in url:
750
778
  # if we found a URL like e.g. http://afafasasfasfas; that makes no
751
779
  # sense, thus skip it, but include http://[2001:db8::1]
@@ -779,9 +807,6 @@ class EmlParser:
779
807
  if url.endswith('://'):
780
808
  return None
781
809
 
782
- if '&' in url:
783
- url = unescape(url)
784
-
785
810
  return url
786
811
 
787
812
  def get_uri_ondata(self, body: str) -> list[str]:
@@ -836,15 +861,26 @@ class EmlParser:
836
861
  _field = eml_parser.decode.workaround_field_value_parsing_errors(self.msg, header)
837
862
  field = []
838
863
 
839
- for v in _field:
840
- v = eml_parser.decode.rfc2047_decode(v).replace('\n', '').replace('\r', '')
864
+ for raw_value in _field:
865
+ v = eml_parser.decode.rfc2047_decode(raw_value).replace('\n', '').replace('\r', '')
841
866
 
842
- parsing_result: dict[str, typing.Any] = {}
843
- parser_cls = typing.cast(email.headerregistry.AddressHeader, email.headerregistry.HeaderRegistry()[header])
844
- parser_cls.parse(v, parsing_result)
845
- for _group in parsing_result['groups']:
846
- for _address in _group.addresses:
847
- field.append((_address.display_name, _address.addr_spec))
867
+ try:
868
+ parsing_result: dict[str, typing.Any] = {}
869
+ parser_cls = typing.cast('email.headerregistry.AddressHeader', email.headerregistry.HeaderRegistry()[header])
870
+ parser_cls.parse(v, parsing_result)
871
+ for _group in parsing_result['groups']:
872
+ for _address in _group.addresses:
873
+ field.append((_address.display_name, _address.addr_spec))
874
+ except (AttributeError, IndexError):
875
+ # The stdlib address/header parser can itself raise on certain malformed
876
+ # values (e.g. an unquoted display name ending in a bare period, such as
877
+ # "Test.<test@example.com>"). The exact exception type raised for this
878
+ # case has varied across Python versions, so we defensively catch it here
879
+ # too and fall back to a best-effort regex extraction of e-mail addresses.
880
+ logger.exception('Field value parsing error while parsing "%s", falling back to regex extraction!', v)
881
+
882
+ for m in eml_parser.regexes.email_regex.findall(v):
883
+ field.append(('', m))
848
884
 
849
885
  return_field = []
850
886
 
@@ -859,23 +895,31 @@ class EmlParser:
859
895
  return return_field
860
896
 
861
897
  def get_raw_body_text(
862
- self, msg: email.message.Message, boundary: typing.Optional[str] = None
863
- ) -> list[tuple[typing.Any, typing.Any, typing.Any, typing.Optional[str]]]:
898
+ self, msg: email.message.Message, boundary: str | None = None, depth: int = 0
899
+ ) -> list[tuple[typing.Any, typing.Any, typing.Any, str | None]]:
864
900
  """This method recursively retrieves all e-mail body parts and returns them as a list.
865
901
 
866
902
  Args:
867
903
  msg (email.message.Message): The actual e-mail message or sub-message.
868
904
  boundary: Used for passing the boundary marker of multipart messages, and used to easier distinguish different parts.
905
+ depth: Parameter used to track the current recursion level.
869
906
 
870
907
  Returns:
871
908
  list: Returns a list of sets which are in the form of "set(encoding, raw_body_string, message field headers, possible boundary marker)"
909
+
910
+ Raises:
911
+ RecursionError: If recursion limit exceeded.
872
912
  """
873
- raw_body: list[tuple[typing.Any, typing.Any, typing.Any, typing.Optional[str]]] = []
913
+ raw_body: list[tuple[typing.Any, typing.Any, typing.Any, str | None]] = []
914
+
915
+ if depth > EmlParser.MULTIPART_RECURSION_LIMIT:
916
+ logger.warning('multi-part nesting limit (%d) exceeded, aborting', EmlParser.MULTIPART_RECURSION_LIMIT)
917
+ raise RecursionError('Recursion limit exceeded')
874
918
 
875
919
  if msg.is_multipart():
876
920
  boundary = msg.get_boundary(failobj=None)
877
921
  for part in msg.get_payload():
878
- raw_body.extend(self.get_raw_body_text(typing.cast(email.message.Message, part), boundary=boundary))
922
+ raw_body.extend(self.get_raw_body_text(typing.cast('email.message.Message', part), boundary=boundary, depth=depth + 1))
879
923
  else:
880
924
  # Treat text document attachments as belonging to the body of the mail.
881
925
  # Attachments with a file-extension of .htm/.html are implicitly treated
@@ -897,14 +941,14 @@ class EmlParser:
897
941
 
898
942
  charset = msg.get_content_charset()
899
943
  if charset is None:
900
- raw_body_b = typing.cast(bytes, msg.get_payload(decode=True))
944
+ raw_body_b = typing.cast('bytes', msg.get_payload(decode=True))
901
945
  raw_body_str = eml_parser.decode.decode_string(raw_body_b, None)
902
946
  else:
903
947
  try:
904
- raw_body_str = typing.cast(bytes, msg.get_payload(decode=True)).decode(charset, 'ignore')
948
+ raw_body_str = typing.cast('bytes', msg.get_payload(decode=True)).decode(charset, 'ignore')
905
949
  except (LookupError, ValueError):
906
950
  logger.debug('An exception occurred while decoding the payload!', exc_info=True)
907
- raw_body_str = typing.cast(bytes, msg.get_payload(decode=True)).decode('ascii', 'ignore')
951
+ raw_body_str = typing.cast('bytes', msg.get_payload(decode=True)).decode('ascii', 'ignore')
908
952
 
909
953
  # In case we hit bug 27257 or any other parsing error, try to downgrade the used policy
910
954
  try:
@@ -931,7 +975,7 @@ class EmlParser:
931
975
  return {k: EmlParser.get_hash(data, k) for k in hashalgo}
932
976
 
933
977
  @staticmethod
934
- def get_hash(value: typing.Union[str, bytes], hash_type: str) -> str:
978
+ def get_hash(value: str | bytes, hash_type: str) -> str:
935
979
  """Generate a hash of type *hash_type* for a given value.
936
980
 
937
981
  Args:
@@ -953,20 +997,28 @@ class EmlParser:
953
997
 
954
998
  return hash_algo(_value).hexdigest()
955
999
 
956
- def traverse_multipart(self, msg: email.message.Message, counter: int = 0) -> dict[str, typing.Any]:
1000
+ def traverse_multipart(self, msg: email.message.Message, counter: int = 0, depth: int = 0) -> dict[str, typing.Any]:
957
1001
  """Recursively traverses all e-mail message multi-part elements and returns in a parsed form as a dict.
958
1002
 
959
1003
  Args:
960
1004
  msg (email.message.Message): An e-mail message object.
961
1005
  counter (int, optional): A counter which is used for generating attachments
962
1006
  file-names in case there are none found in the header. Default = 0.
1007
+ depth: Parameter used to track the current recursion level.
963
1008
 
964
1009
  Returns:
965
1010
  dict: Returns a dict with all original multi-part headers as well as generated hash check-sums,
966
1011
  date size, file extension, real mime-type.
1012
+
1013
+ Raises:
1014
+ RecursionError: If recursion limit exceeded.
967
1015
  """
968
1016
  attachments = {}
969
1017
 
1018
+ if depth > EmlParser.MULTIPART_RECURSION_LIMIT:
1019
+ logger.warning('multi-part nesting limit (%d) exceeded, aborting', EmlParser.MULTIPART_RECURSION_LIMIT)
1020
+ raise RecursionError('Recursion limit exceeded')
1021
+
970
1022
  if msg.is_multipart():
971
1023
  if 'content-type' in msg:
972
1024
  if msg.get_content_type() == 'message/rfc822':
@@ -974,7 +1026,7 @@ class EmlParser:
974
1026
  attachments.update(self.prepare_multipart_part_attachment(msg, counter))
975
1027
 
976
1028
  for part in msg.get_payload():
977
- attachments.update(self.traverse_multipart(typing.cast(email.message.EmailMessage, part), counter))
1029
+ attachments.update(self.traverse_multipart(typing.cast('email.message.EmailMessage', part), counter=counter, depth=depth + 1))
978
1030
  else:
979
1031
  return self.prepare_multipart_part_attachment(msg, counter)
980
1032
 
@@ -1013,14 +1065,14 @@ class EmlParser:
1013
1065
 
1014
1066
  try:
1015
1067
  custom_policy: email.policy.Policy = email.policy.default.clone(max_line_length=0)
1016
- data = typing.cast(list[email.message.EmailMessage], payload)[0].as_bytes(policy=custom_policy)
1068
+ data = typing.cast('list[email.message.EmailMessage]', payload)[0].as_bytes(policy=custom_policy)
1017
1069
  except UnicodeEncodeError:
1018
1070
  custom_policy = email.policy.compat32.clone(max_line_length=0)
1019
- data = typing.cast(list[email.message.EmailMessage], payload)[0].as_bytes(policy=custom_policy)
1071
+ data = typing.cast('list[email.message.EmailMessage]', payload)[0].as_bytes(policy=custom_policy)
1020
1072
 
1021
1073
  file_size = len(data)
1022
1074
  else:
1023
- data = typing.cast(bytes, msg.get_payload(decode=True))
1075
+ data = typing.cast('bytes', msg.get_payload(decode=True))
1024
1076
  file_size = len(data)
1025
1077
 
1026
1078
  filename = msg.get_filename('')
@@ -1055,9 +1107,9 @@ class EmlParser:
1055
1107
  attachment[file_id]['raw'] = base64.b64encode(data)
1056
1108
 
1057
1109
  ch: dict[str, list[str]] = {}
1058
- for k, v in msg.items():
1059
- k = k.lower()
1060
- v = str(v)
1110
+ for header_key, header_value in msg.items():
1111
+ k = header_key.lower()
1112
+ v = str(header_value)
1061
1113
 
1062
1114
  if k in ch:
1063
1115
  ch[k].append(v)
@@ -1071,7 +1123,7 @@ class EmlParser:
1071
1123
  return attachment
1072
1124
 
1073
1125
  @staticmethod
1074
- def get_mime_type(data: bytes) -> typing.Union[tuple[str, str], tuple[None, None]]:
1126
+ def get_mime_type(data: bytes) -> tuple[str, str] | tuple[None, None]:
1075
1127
  """Get mime-type information based on the provided bytes object.
1076
1128
 
1077
1129
  Args:
@@ -14,8 +14,12 @@ __license__ = 'AGPL v3+'
14
14
 
15
15
  # regex compilation
16
16
  # W3C HTML5 standard recommended regex for e-mail validation
17
- email_regex = re.compile(r"""([a-zA-Z0-9.!#$%&'*+-/=?^_`{|}~-]+@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*)""", re.MULTILINE)
18
- email_force_tld_regex = re.compile(r"""([a-zA-Z0-9.!#$%&'*+-/=?^_`{|}~-]+@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)+)""", re.MULTILINE)
17
+ # It also implements the 64 character limit on the local part (RFC 5321, section 2.4), with a safety factor.
18
+ # This mitigates an issue with quadratic complexity when searching for e-mail addresses in very long strings.
19
+ email_no_force_tld_regex = re.compile(r"""([a-zA-Z0-9.!#$%&'*+\-/=?^_`{|}~]{1,128}@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*)""", re.MULTILINE)
20
+ email_force_tld_regex = re.compile(r"""([a-zA-Z0-9.!#$%&'*+\-/=?^_`{|}~]{1,128}@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)+)""", re.MULTILINE)
21
+ email_regex = email_no_force_tld_regex
22
+
19
23
 
20
24
  # regex for detecting RFC2047 encodings - used from https://dmorgan.info/posts/encoded-word-syntax/
21
25
  email_regex_rfc2047 = re.compile(r"""=\?{1}([\w\S]+)\?{1}([B|Q|b|q])\?{1}([\w\S]+)\?{1}=""")
@@ -96,7 +100,6 @@ url_regex_href = re.compile(
96
100
  )
97
101
 
98
102
  date_regex = re.compile(r""";[ \w\s:,+\-()]+$""")
99
- noparenthesis_regex = re.compile(r"""\([^()]*\)""")
100
103
  cleanline_regex = re.compile(r"""(^[;\s]{0,}|[;\s]{0,}$)""")
101
104
 
102
105
  escape_special_regex_chars = re.compile(r"""([\^$\[\]()+?.])""")
@@ -8,29 +8,24 @@ import eml_parser.regexes
8
8
 
9
9
 
10
10
  def noparenthesis(line: str) -> str:
11
- """Remove nested parenthesis, until none are present.
12
-
13
- @FIXME rewrite this function.
11
+ """Remove nested parentheses, until no pairs of matching parentyheses are present.
14
12
 
15
13
  Args:
16
- line (str): Input text to search in for parenthesis.
14
+ line (str): Input text to search in for parentheses.
17
15
 
18
16
  Returns:
19
- str: Return a string with all parenthesis removed.
17
+ str: Return a string with all text in matching parentheses removed. For example,
18
+ noparenthesis('a((b)c(d') == 'a(c(d'
20
19
  """
21
- # check empty string
22
- if not line:
23
- return line
24
-
25
- line_ = line
26
-
27
- while True:
28
- lline = line_
29
- line_ = eml_parser.regexes.noparenthesis_regex.sub('', line_)
30
- if lline == line_:
31
- break
32
-
33
- return line_
20
+ fragments: list[list[str]] = [[]]
21
+ for ch in line:
22
+ if ch == '(':
23
+ fragments.append([ch])
24
+ else:
25
+ fragments[-1].append(ch)
26
+ if ch == ')' and len(fragments) > 1:
27
+ fragments.pop()
28
+ return ''.join(ch for w in fragments for ch in w)
34
29
 
35
30
 
36
31
  def cleanline(line: str) -> str:
@@ -63,7 +58,7 @@ def get_domain_ip(line: str) -> list[str]:
63
58
 
64
59
 
65
60
  def parserouting(line: str) -> dict[str, typing.Any]:
66
- """This method tries to parsed a e-mail header received line and extract machine readable information.
61
+ """This method tries to parse an e-mail header received line and extract machine-readable information.
67
62
 
68
63
  Note that there are a large number of formats for these lines
69
64
  and a lot of weird ones which are not commonly used.
@@ -105,49 +100,32 @@ def parserouting(line: str) -> dict[str, typing.Any]:
105
100
  npline = npline.strip(' ') # Remove any border WhiteSpace
106
101
 
107
102
  borders = ['from ', 'by ', 'with ', 'for ']
108
- result: list[dict[str, typing.Any]] = []
109
103
 
110
104
  # Scan the line to determine the order, and presence of each "from/by/with/for" words
105
+ raw_matches: list[tuple[int, str]] = []
111
106
  for word in borders:
112
- candidate = list(borders)
113
- candidate.remove(word)
114
- for endword in candidate:
115
- if word in npline:
116
- loc = npline.find(word)
117
- end = npline.find(endword)
118
- if end < loc or end == -1:
119
- end = 0xFFFFFFF # Kindof MAX 31 bits
120
- result.append({'name_in': word, 'pos': loc, 'name_out': endword, 'weight': end + loc})
121
- # print({'name_in': word, 'pos': loc, 'name_out': endword, 'weight': end+loc})
122
-
123
- # Create the word list... "from/by/with/for" by sorting the list.
124
- if not result:
107
+ pos = npline.find(word)
108
+ if pos != -1:
109
+ raw_matches.append((pos, word))
110
+
111
+ if not raw_matches:
125
112
  out['warning'] = ['Nothing Parsable']
126
113
  return out
127
114
 
128
- tout = []
129
- for word in borders:
130
- result_max = 0xFFFFFFFF
131
- line_max: dict[str, typing.Any] = {}
132
- for eline in result:
133
- if eline['name_in'] == word and eline['weight'] <= result_max:
134
- result_max = eline['weight']
135
- line_max = eline
115
+ # Sort keywords sequentially by their appearance order in the header
116
+ tout = sorted(raw_matches, key=lambda match: match[0])
136
117
 
137
- if line_max:
138
- tout.append([line_max.get('pos'), line_max.get('name_in')])
118
+ reg = ''
119
+ for i, item in enumerate(tout):
120
+ field = item[1].strip()
121
+ kw = item[1]
139
122
 
140
- # structure is list[list[int, str]]
141
- # we sort based on the first element of the sub list, i.e. int
142
- tout = sorted(tout, key=lambda x: typing.cast(int, x[0]))
123
+ # Capture lazily until the next keyword
124
+ reg += kw + r'(?P<' + field + r'>.*?)'
143
125
 
144
- # build regex.
145
- reg = ''
146
- for item in tout:
147
- reg += item[1] + '(?P<' + item[1].strip() + '>.*)' # type: ignore
148
- if npdate:
149
- # escape special regex chars
150
- reg += eml_parser.regexes.escape_special_regex_chars.sub(r"""\\\1""", npdate)
126
+ if i + 1 >= len(tout):
127
+ # Last field captures lazily until a date separator ';' or end of string
128
+ reg += '(?:;|$)'
151
129
 
152
130
  reparse = re.compile(reg)
153
131
  reparseg = reparse.search(line)