mail-parser 4.0.0__tar.gz → 4.1.2__tar.gz

{mail_parser-4.0.0/src/mail_parser.egg-info → mail_parser-4.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mail-parser
-Version: 4.0.0
+Version: 4.1.2
 Summary: Improved wrapper for email standard library
 Home-page: https://github.com/SpamScope/mail-parser
 Author: Fedele Mantuano
@@ -37,8 +37,10 @@ Requires-Dist: pytest-cov; extra == "test"
 Requires-Dist: pytest-mock; extra == "test"
 Requires-Dist: pytest-ordering; extra == "test"
 
-[![PyPI version](https://badge.fury.io/py/mail-parser.svg)](https://badge.fury.io/py/mail-parser)
+[![PyPI - Version](https://img.shields.io/pypi/v/mail-parser)](https://pypi.org/project/mail-parser/)
 [![Coverage Status](https://coveralls.io/repos/github/SpamScope/mail-parser/badge.svg?branch=develop)](https://coveralls.io/github/SpamScope/mail-parser?branch=develop)
+[![PyPI - Downloads](https://img.shields.io/pypi/dm/mail-parser?color=blue)](https://pypistats.org/packages/mail-parser)
+
 
 ![SpamScope](https://raw.githubusercontent.com/SpamScope/spamscope/develop/docs/logo/spamscope.png)
 
@@ -67,9 +69,16 @@ mail-parser supports Python 3.
 mail-parser can be downloaded, used, and modified free of charge. It is available under the Apache 2 license.
 
 ## Support the project
+If you find this project useful, you can support it by donating any amount you want. All donations are greatly appreciated and help maintain and develop the project.
 
 [![Donate](https://www.paypal.com/en_US/i/btn/btn_donateCC_LG.gif "Donate")](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=VEPXYP745KJF2)
 
+<a href="bitcoin:bc1qxhz3tghztpjqdt7atey68s344wvmugtl55tm32">
+  <img src="https://github.com/SpamScope/mail-parser/blob/develop/docs/images/Bitcoin%20SpamScope.jpg?raw=true" alt="Bitcoin" width="200">
+</a>
+
+Bitcoin Address: `bc1qxhz3tghztpjqdt7atey68s344wvmugtl55tm32`
+
 # mail-parser on Web
  - [Splunk app](https://splunkbase.splunk.com/app/4129/)
  - [FreeBSD port](https://www.freshports.org/mail/py-mail-parser/)
@@ -318,7 +327,7 @@ $ pip install -e ".[dev, test]"
 The second step is to run the tests:
 
 ```
-$ make test
+$ make unittest
 ```
 
 Then you can try to run the command line tool:

{mail_parser-4.0.0 → mail_parser-4.1.2}/README.md

@@ -1,5 +1,7 @@
-[![PyPI version](https://badge.fury.io/py/mail-parser.svg)](https://badge.fury.io/py/mail-parser)
+[![PyPI - Version](https://img.shields.io/pypi/v/mail-parser)](https://pypi.org/project/mail-parser/)
 [![Coverage Status](https://coveralls.io/repos/github/SpamScope/mail-parser/badge.svg?branch=develop)](https://coveralls.io/github/SpamScope/mail-parser?branch=develop)
+[![PyPI - Downloads](https://img.shields.io/pypi/dm/mail-parser?color=blue)](https://pypistats.org/packages/mail-parser)
+
 
 ![SpamScope](https://raw.githubusercontent.com/SpamScope/spamscope/develop/docs/logo/spamscope.png)
 
@@ -28,9 +30,16 @@ mail-parser supports Python 3.
 mail-parser can be downloaded, used, and modified free of charge. It is available under the Apache 2 license.
 
 ## Support the project
+If you find this project useful, you can support it by donating any amount you want. All donations are greatly appreciated and help maintain and develop the project.
 
 [![Donate](https://www.paypal.com/en_US/i/btn/btn_donateCC_LG.gif "Donate")](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=VEPXYP745KJF2)
 
+<a href="bitcoin:bc1qxhz3tghztpjqdt7atey68s344wvmugtl55tm32">
+  <img src="https://github.com/SpamScope/mail-parser/blob/develop/docs/images/Bitcoin%20SpamScope.jpg?raw=true" alt="Bitcoin" width="200">
+</a>
+
+Bitcoin Address: `bc1qxhz3tghztpjqdt7atey68s344wvmugtl55tm32`
+
 # mail-parser on Web
  - [Splunk app](https://splunkbase.splunk.com/app/4129/)
  - [FreeBSD port](https://www.freshports.org/mail/py-mail-parser/)
@@ -279,7 +288,7 @@ $ pip install -e ".[dev, test]"
 The second step is to run the tests:
 
 ```
-$ make test
+$ make unittest
 ```
 
 Then you can try to run the command line tool:
@@ -288,4 +297,4 @@ Then you can try to run the command line tool:
 $ mail-parser -f tests/mails/mail_malformed_3 -j
 ```
 
-If all is ok, you can start to develop.
+If all is ok, you can start to develop.

{mail_parser-4.0.0 → mail_parser-4.1.2}/pyproject.toml

@@ -1,3 +1,3 @@
 [build-system]
 requires = ["setuptools >= 40.6.0", "wheel"]
-build-backend = "setuptools.build_meta"
+build-backend = "setuptools.build_meta"

{mail_parser-4.0.0 → mail_parser-4.1.2}/setup.py

@@ -17,4 +17,4 @@ limitations under the License.
 import setuptools
 
 if __name__ == "__main__":
-    setuptools.setup()
+    setuptools.setup()

{mail_parser-4.0.0 → mail_parser-4.1.2/src/mail_parser.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mail-parser
-Version: 4.0.0
+Version: 4.1.2
 Summary: Improved wrapper for email standard library
 Home-page: https://github.com/SpamScope/mail-parser
 Author: Fedele Mantuano
@@ -37,8 +37,10 @@ Requires-Dist: pytest-cov; extra == "test"
 Requires-Dist: pytest-mock; extra == "test"
 Requires-Dist: pytest-ordering; extra == "test"
 
-[![PyPI version](https://badge.fury.io/py/mail-parser.svg)](https://badge.fury.io/py/mail-parser)
+[![PyPI - Version](https://img.shields.io/pypi/v/mail-parser)](https://pypi.org/project/mail-parser/)
 [![Coverage Status](https://coveralls.io/repos/github/SpamScope/mail-parser/badge.svg?branch=develop)](https://coveralls.io/github/SpamScope/mail-parser?branch=develop)
+[![PyPI - Downloads](https://img.shields.io/pypi/dm/mail-parser?color=blue)](https://pypistats.org/packages/mail-parser)
+
 
 ![SpamScope](https://raw.githubusercontent.com/SpamScope/spamscope/develop/docs/logo/spamscope.png)
 
@@ -67,9 +69,16 @@ mail-parser supports Python 3.
 mail-parser can be downloaded, used, and modified free of charge. It is available under the Apache 2 license.
 
 ## Support the project
+If you find this project useful, you can support it by donating any amount you want. All donations are greatly appreciated and help maintain and develop the project.
 
 [![Donate](https://www.paypal.com/en_US/i/btn/btn_donateCC_LG.gif "Donate")](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=VEPXYP745KJF2)
 
+<a href="bitcoin:bc1qxhz3tghztpjqdt7atey68s344wvmugtl55tm32">
+  <img src="https://github.com/SpamScope/mail-parser/blob/develop/docs/images/Bitcoin%20SpamScope.jpg?raw=true" alt="Bitcoin" width="200">
+</a>
+
+Bitcoin Address: `bc1qxhz3tghztpjqdt7atey68s344wvmugtl55tm32`
+
 # mail-parser on Web
  - [Splunk app](https://splunkbase.splunk.com/app/4129/)
  - [FreeBSD port](https://www.freshports.org/mail/py-mail-parser/)
@@ -318,7 +327,7 @@ $ pip install -e ".[dev, test]"
 The second step is to run the tests:
 
 ```
-$ make test
+$ make unittest
 ```
 
 Then you can try to run the command line tool:

{mail_parser-4.0.0 → mail_parser-4.1.2}/src/mail_parser.egg-info/SOURCES.txt

@@ -1,5 +1,4 @@
 LICENSE.txt
-MANIFEST.in
 NOTICE.txt
 README.md
 pyproject.toml
@@ -14,8 +13,8 @@ src/mail_parser.egg-info/top_level.txt
 src/mailparser/__init__.py
 src/mailparser/__main__.py
 src/mailparser/const.py
+src/mailparser/core.py
 src/mailparser/exceptions.py
-src/mailparser/mailparser.py
 src/mailparser/utils.py
 src/mailparser/version.py
 tests/test_mail_parser.py

{mail_parser-4.0.0 → mail_parser-4.1.2}/src/mailparser/__init__.py

@@ -17,13 +17,23 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 
-
-from .mailparser import (
+from mailparser.core import (
     MailParser,
     parse_from_bytes,
     parse_from_file,
     parse_from_file_msg,
     parse_from_file_obj,
-    parse_from_string)
+    parse_from_string,
+)
+
+from mailparser.utils import get_header
 
-from .utils import get_header
+__all__ = [
+    "MailParser",
+    "parse_from_bytes",
+    "parse_from_file",
+    "parse_from_file_msg",
+    "parse_from_file_obj",
+    "parse_from_string",
+    "get_header",
+]

mail_parser-4.1.2/src/mailparser/__main__.py

@@ -0,0 +1,348 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Copyright 2016 Fedele Mantuano (https://twitter.com/fedelemantuano)
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import argparse
+import logging
+import sys
+
+import mailparser
+from mailparser.exceptions import MailParserOutlookError
+from mailparser.utils import (
+    custom_log,
+    print_attachments,
+    print_mail_fingerprints,
+    safe_print,
+    write_attachments,
+)
+from mailparser.version import __version__
+
+
+log = logging.getLogger("mailparser")
+
+
+def get_args():
+    """
+    Get arguments from command line.
+    :return: argparse.ArgumentParser
+    :rtype: argparse.ArgumentParser
+    """
+    parser = argparse.ArgumentParser(
+        description="Wrapper for email Python Standard Library",
+        epilog="It takes as input a raw mail and generates a parsed object.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+
+    parsing_group = parser.add_mutually_exclusive_group(required=True)
+    parsing_group.add_argument("-f", "--file", dest="file", help="Raw email file")
+    parsing_group.add_argument("-s", "--string", dest="string", help="Raw email string")
+    parsing_group.add_argument(
+        "-k",
+        "--stdin",
+        dest="stdin",
+        action="store_true",
+        help="Enable parsing from stdin",
+    )
+
+    parser.add_argument(
+        "-l",
+        "--log-level",
+        dest="log_level",
+        default="WARNING",
+        choices=["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET"],
+        help="Set log level",
+    )
+
+    parser.add_argument(
+        "-j",
+        "--json",
+        dest="json",
+        action="store_true",
+        help="Show the JSON of parsed mail",
+    )
+
+    parser.add_argument(
+        "-b", "--body", dest="body", action="store_true", help="Print the body of mail"
+    )
+
+    parser.add_argument(
+        "-a",
+        "--attachments",
+        dest="attachments",
+        action="store_true",
+        help="Print the attachments of mail",
+    )
+
+    parser.add_argument(
+        "-r",
+        "--headers",
+        dest="headers",
+        action="store_true",
+        help="Print the headers of mail",
+    )
+
+    parser.add_argument(
+        "-t", "--to", dest="to", action="store_true", help="Print the to of mail"
+    )
+
+    parser.add_argument(
+        "-dt",
+        "--delivered-to",
+        dest="delivered_to",
+        action="store_true",
+        help="Print the delivered-to of mail",
+    )
+
+    parser.add_argument(
+        "-m", "--from", dest="from_", action="store_true", help="Print the from of mail"
+    )
+
+    parser.add_argument(
+        "-u",
+        "--subject",
+        dest="subject",
+        action="store_true",
+        help="Print the subject of mail",
+    )
+
+    parser.add_argument(
+        "-c",
+        "--receiveds",
+        dest="receiveds",
+        action="store_true",
+        help="Print all receiveds of mail",
+    )
+
+    parser.add_argument(
+        "-d",
+        "--defects",
+        dest="defects",
+        action="store_true",
+        help="Print the defects of mail",
+    )
+
+    parser.add_argument(
+        "-o",
+        "--outlook",
+        dest="outlook",
+        action="store_true",
+        help="Analyze Outlook msg",
+    )
+
+    parser.add_argument(
+        "-i",
+        "--senderip",
+        dest="senderip",
+        metavar="Trust mail server string",
+        help="Extract a reliable sender IP address heuristically",
+    )
+
+    parser.add_argument(
+        "-p",
+        "--mail-hash",
+        dest="mail_hash",
+        action="store_true",
+        help="Print mail fingerprints without headers",
+    )
+
+    parser.add_argument(
+        "-z",
+        "--attachments-hash",
+        dest="attachments_hash",
+        action="store_true",
+        help="Print attachments with fingerprints",
+    )
+
+    parser.add_argument(
+        "-sa",
+        "--store-attachments",
+        dest="store_attachments",
+        action="store_true",
+        help="Store attachments on disk",
+    )
+
+    parser.add_argument(
+        "-ap",
+        "--attachments-path",
+        dest="attachments_path",
+        default="/tmp",
+        help="Path where store attachments",
+    )
+
+    parser.add_argument(
+        "-v", "--version", action="version", version="%(prog)s {}".format(__version__)
+    )
+
+    return parser
+
+
+def main():
+    """
+    Main function.
+    """
+    args = get_args().parse_args()
+    log = custom_log(level=args.log_level, name="mailparser")
+
+    try:
+        parser = get_parser(args)
+        process_output(args, parser)
+    except Exception as e:
+        log.error(f"An error occurred: {e}")
+        sys.exit(1)
+
+
+def get_parser(args):
+    """
+    Get the correct parser based on the input source.
+    :param args: argparse.Namespace
+    :type args: argparse.Namespace
+    :return: MailParser
+    :rtype: mailparser.core.MailParser
+    """
+    if args.file:
+        return parse_file(args)
+    elif args.string:
+        log.debug("Start analysis by string mail")
+        return mailparser.parse_from_string(args.string)
+    elif args.stdin:
+        return parse_stdin(args)
+    else:
+        raise ValueError("No input source provided")
+
+
+def parse_file(args):
+    """
+    Parse the file based on the arguments provided.
+    :param args: argparse.Namespace
+    :type args: argparse.Namespace
+    :return: MailParser
+    :rtype: mailparser.core.MailParser
+    """
+    log.debug("Start analysis by file mail")
+    if args.outlook:
+        log.debug("Start analysis by Outlook msg")
+        return mailparser.parse_from_file_msg(args.file)
+    else:
+        log.debug("Start analysis by raw mail")
+        return mailparser.parse_from_file(args.file)
+
+
+def parse_stdin(args):
+    """
+    Parse the stdin based on the arguments provided.
+    :param args: argparse.Namespace
+    :type args: argparse.Namespace
+    :return: MailParser
+    :rtype: mailparser.core.MailParser
+    """
+    log.debug("Start analysis by stdin mail")
+    if args.outlook:
+        raise MailParserOutlookError("You can't use stdin with msg Outlook")
+    return mailparser.parse_from_file_obj(sys.stdin)
+
+
+def process_output(args, parser):
+    """
+    Process the output based on the arguments provided.
+    :param args: argparse.Namespace
+    :type args: argparse.Namespace
+    :param parser: MailParser
+    :type parser: mailparser.core.MailParser
+    :param log: logger
+    :type log: logging.Logger
+    """
+    if args.json:
+        safe_print(parser.mail_json)
+
+    if args.body:
+        safe_print(parser.body)
+
+    if args.headers:
+        safe_print(parser.headers_json)
+
+    if args.to:
+        safe_print(parser.to_json)
+
+    if args.delivered_to:
+        safe_print(parser.delivered_to_json)
+
+    if args.from_:
+        safe_print(parser.from_json)
+
+    if args.subject:
+        safe_print(parser.subject)
+
+    if args.receiveds:
+        safe_print(parser.received_json)
+
+    if args.defects:
+        print_defects(parser)
+
+    if args.senderip:
+        print_sender_ip(parser, args)
+
+    if args.attachments or args.attachments_hash:
+        print_attachments_details(parser, args)
+
+    if args.mail_hash:
+        log.debug("Printing also mail fingerprints")
+        print_mail_fingerprints(parser.body.encode("utf-8"))
+
+    if args.store_attachments:
+        log.debug("Store attachments on disk")
+        write_attachments(parser.attachments, args.attachments_path)
+
+
+def print_defects(parser):
+    """
+    Print email defects.
+    :param parser: MailParser
+    :type parser: mailparser.core.MailParser
+    """
+    log.debug("Printing defects")
+    for defect in parser.defects_categories:
+        safe_print(defect)
+
+
+def print_sender_ip(parser, args):
+    """
+    Print sender IP address.
+    :param parser: MailParser
+    :type parser: mailparser.core.MailParser
+    :param args: argparse.Namespace
+    :type args: argparse.Namespace
+    """
+    log.debug("Printing sender IP")
+    sender_ip = parser.get_server_ipaddress(args.senderip)
+    safe_print(sender_ip if sender_ip else "Not Found")
+
+
+def print_attachments_details(parser, args):
+    """
+    Print attachments details.
+    :param parser: MailParser
+    :type parser: mailparser.core.MailParser
+    :param args: argparse.Namespace
+    :type args: argparse.Namespace
+    """
+    log.debug("Printing attachments details")
+    print_attachments(parser.attachments, args.attachments_hash)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    main()

mail_parser-4.1.2/src/mailparser/const.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Copyright 2018 Fedele Mantuano (https://twitter.com/fedelemantuano)
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import re
+
+
+REGXIP = re.compile(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}")
+
+JUNK_PATTERN = r"[ \(\)\[\]\t\n]+"
+
+# Patterns for receiveds
+RECEIVED_PATTERNS = [
+    # each pattern handles matching a single clause
+    # need to exclude withs followed by cipher (e.g., google); (?! cipher)
+    # TODO: ideally would do negative matching for with in parens
+    # need the beginning or space to differentiate from envelope-from
+    (
+        r"(?:(?:^|\s)from\s+(?P<from>.+?)(?:\s*[(]?"
+        r"envelope-from|\s*[(]?envelope-sender|\s+"
+        r"by|\s+with(?! cipher)|\s+id|\s+for|\s+via|;))"
+    ),
+    # need to make sure envelope-from comes before from to prevent mismatches
+    # envelope-from and -sender seem to optionally have space and/or
+    # ( before them other clauses must have whitespace before
+    (
+        r"(?:[^-]by\s+(?P<by>.+?)(?:\s*[(]?envelope-from|\s*"
+        r"[(]?envelope-sender|\s+from|\s+with"
+        r"(?! cipher)|\s+id|\s+for|\s+via|;))"
+    ),
+    (
+        r"(?:with(?! cipher)\s+(?P<with>.+?)(?:\s*[(]?envelope-from|\s*[(]?"
+        r"envelope-sender|\s+from|\s+by|\s+id|\s+for|\s+via|;))"
+    ),
+    (
+        r"[^\w](?:id\s+(?P<id>.+?)(?:\s*[(]?envelope-from|\s*"
+        r"[(]?envelope-sender|\s+from|\s+by|\s+with"
+        r"(?! cipher)|\s+for|\s+via|;))"
+    ),
+    (
+        r"(?:for\s+(?P<for>.+?)(?:\s*[(]?envelope-from|\s*[(]?"
+        r"envelope-sender|\s+from|\s+by|\s+with"
+        r"(?! cipher)|\s+id|\s+via|;))"
+    ),
+    (
+        r"(?:via\s+(?P<via>.+?)(?:\s*[(]?"
+        r"envelope-from|\s*[(]?envelope-sender|\s+"
+        r"from|\s+by|\s+id|\s+for|\s+with(?! cipher)|;))"
+    ),
+    # assumes emails are always inside <>
+    r"(?:envelope-from\s+<(?P<envelope_from>.+?)>)",
+    r"(?:envelope-sender\s+<(?P<envelope_sender>.+?)>)",
+    # datetime comes after ; at the end
+    r";\s*(?P<date>.*)",
+    # sendgrid datetime
+    (
+        r"(?P<date>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:"
+        r"\d{2}\.\d{9} \+0000 UTC) m=\+\d+\.\d+"
+    ),
+]
+
+RECEIVED_COMPILED_LIST = [re.compile(i, re.I | re.DOTALL) for i in RECEIVED_PATTERNS]
+
+EPILOGUE_DEFECTS = {"StartBoundaryNotFoundDefect"}
+
+ADDRESSES_HEADERS = set(["bcc", "cc", "delivered-to", "from", "reply-to", "to"])
+
+# These parts are always returned
+OTHERS_PARTS = set(
+    [
+        "attachments",
+        "body",
+        "date",
+        "message-id",
+        "received",
+        "subject",
+        "timezone",
+        "to_domains",
+        "user-agent",
+        "x-mailer",
+        "x-original-to",
+    ]
+)