mailsort 0.0.1__py3-none-any.whl

mailsort/__init__.py

@@ -0,0 +1,6 @@
+from mailsort.local import Imap
+
+from . import _version
+
+__version__: str = _version.__version__
+__all__ = ["Imap"]

mailsort/__main__.py

@@ -0,0 +1,99 @@
+import argparse
+import os
+
+from mailsort import Imap
+
+
+def command_line_parser():
+    """
+    Main function primarily used for the command line interface of the IMAP backend
+    """
+    parser = argparse.ArgumentParser(prog="mailsort")
+    parser.add_argument(
+        "--host",
+        help="IMAP server hostname e.g. imap.example.com .",
+    )
+    parser.add_argument(
+        "--port",
+        type=int,
+        default=993,
+        help="IMAP server port - default: 993 .",
+    )
+    parser.add_argument(
+        "--username",
+        help="IMAP account username.",
+    )
+    parser.add_argument(
+        "--password-env",
+        default="IMAP_PASSWORD",
+        help=(
+            "Name of the environment variable holding the IMAP account password - "
+            "default: IMAP_PASSWORD ."
+        ),
+    )
+    parser.add_argument(
+        "--no-ssl",
+        action="store_true",
+        help="Connect without SSL (IMAP4 instead of IMAP4_SSL).",
+    )
+    parser.add_argument(
+        "-d",
+        "--database",
+        help="Connection string to connect to database e.g. sqlite:///email.db .",
+    )
+    parser.add_argument(
+        "-u",
+        "--update",
+        action="store_true",
+        help="Update local database and retrain machine learning model.",
+    )
+    parser.add_argument(
+        "-i",
+        "--identification",
+        help="User ID of the database user e.g. 1 .",
+    )
+    parser.add_argument(
+        "-l",
+        "--label",
+        help="Email label (IMAP folder) to be filtered with machine learning.",
+    )
+    args = parser.parse_args()
+    db_user_id = int(args.identification) if args.identification else 1
+    password = os.environ.get(args.password_env)
+    if not args.host or not args.username:
+        print("Please provide --host and --username.")
+    elif not password:
+        print(
+            f"Please set the {args.password_env} environment variable to your IMAP password."
+        )
+    else:
+        database = args.database or "sqlite:///email.db"
+        imap = Imap(
+            host=args.host,
+            port=args.port,
+            username=args.username,
+            password=password,
+            connection_str=database,
+            db_user_id=db_user_id,
+            use_ssl=not args.no_ssl,
+            email_download_format="metadata",
+        )
+        if args.update:
+            imap.update_database(quick=False)
+            imap.fit_machine_learning_model_to_database(
+                n_estimators=100,
+                max_features=400,
+                random_state=42,
+                bootstrap=True,
+                include_deleted=False,
+            )
+        elif args.label:
+            imap.filter_messages_from_server(
+                label=args.label, recommendation_ratio=0.9, label_prefix="labels_"
+            )
+        else:
+            parser.print_help()
+
+
+if __name__ == "__main__":
+    command_line_parser()

mailsort/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.0.1'
+__version_tuple__ = version_tuple = (0, 0, 1)
+
+__commit_id__ = commit_id = None

mailsort/base/__init__.py

@@ -0,0 +1,3 @@
+from mailsort.base.database import get_email_database
+
+__all__ = ["get_email_database"]

mailsort/base/database.py

@@ -0,0 +1,465 @@
+from typing import Any
+
+import pandas
+from sqlalchemy import Boolean, Column, DateTime, Engine, ForeignKey, Integer, String
+from sqlalchemy.orm import Session, declarative_base
+from tqdm import tqdm
+
+Base = declarative_base()
+
+
+class EmailContent(Base):
+    __tablename__ = "email_content"
+    id = Column(Integer, primary_key=True)
+    email_id = Column(String)
+    email_subject = Column(String)
+    email_content = Column(String)
+    email_deleted = Column(Boolean)
+    email_date = Column(DateTime)
+    user_id = Column(Integer)
+
+
+class Threads(Base):
+    __tablename__ = "email_threads"
+    id = Column(Integer, primary_key=True)
+    email_id = Column(String, ForeignKey("email_content.email_id"))
+    thread_id = Column(String)
+    user_id = Column(Integer)
+
+
+class Labels(Base):
+    __tablename__ = "email_labels"
+    id = Column(Integer, primary_key=True)
+    email_id = Column(String, ForeignKey("email_content.email_id"))
+    label_id = Column(String)
+    user_id = Column(Integer)
+
+
+class EmailTo(Base):
+    __tablename__ = "email_to"
+    id = Column(Integer, primary_key=True)
+    email_id = Column(String, ForeignKey("email_content.email_id"))
+    email_to = Column(String)
+    user_id = Column(Integer)
+
+
+class EmailCc(Base):
+    __tablename__ = "email_cc"
+    id = Column(Integer, primary_key=True)
+    email_id = Column(String, ForeignKey("email_content.email_id"))
+    email_cc = Column(String)
+    user_id = Column(Integer)
+
+
+class EmailFrom(Base):
+    __tablename__ = "email_from"
+    id = Column(Integer, primary_key=True)
+    email_id = Column(String, ForeignKey("email_content.email_id"))
+    email_from = Column(String)
+    user_id = Column(Integer)
+
+
+class DatabaseTemplate:
+    def __init__(self, session: Session) -> None:
+        self._session = session
+
+    def close(self) -> None:
+        self._session.close()
+
+
+class DatabaseInterface(DatabaseTemplate):
+    @property
+    def session(self) -> Session:
+        return self._session
+
+    def store_dataframe(self, df: pandas.DataFrame, user_id: int = 1) -> None:
+        self._commit_content_table(df=df, user_id=user_id)
+        self._commit_email_from_table(df=df, user_id=user_id)
+        self._commit_email_to_table(df=df, user_id=user_id)
+        self._commit_email_cc_table(df=df, user_id=user_id)
+        self._commit_label_table(df=df, user_id=user_id)
+        self._commit_thread_table(df=df, user_id=user_id)
+
+    def list_email_ids(self, user_id: int = 1) -> list[str]:
+        return [
+            instance.email_id
+            for instance in self._session.query(EmailContent)
+            .filter(EmailContent.user_id == user_id)
+            .order_by(EmailContent.id)
+        ]
+
+    def mark_emails_as_deleted(
+        self, message_id_lst: list[str], user_id: int = 1
+    ) -> None:
+        for instance in (
+            self._session.query(EmailContent)
+            .filter(EmailContent.user_id == user_id)
+            .filter(EmailContent.email_id.in_(message_id_lst))
+            .all()
+        ):
+            instance.email_deleted = True
+        self._session.commit()
+
+    def get_labels_to_update(
+        self, message_id_lst: list[str], user_id: int = 1
+    ) -> tuple[list[str], list[str], list[str]]:
+        email_in_db_id = self.list_email_ids(user_id=user_id)
+        new_messages_lst = [m for m in message_id_lst if m not in email_in_db_id]
+        deleted_messages_lst = [m for m in email_in_db_id if m not in message_id_lst]
+        message_label_updates_lst = [m for m in message_id_lst if m in email_in_db_id]
+        return new_messages_lst, message_label_updates_lst, deleted_messages_lst
+
+    def update_labels(
+        self,
+        message_id_lst: list[str],
+        message_meta_lst: list[list[str]],
+        user_id: int = 1,
+    ) -> None:
+        for message_id, message_labels in tqdm(
+            iterable=zip(message_id_lst, message_meta_lst, strict=False),
+            desc="Update labels",
+            total=len(message_id_lst),
+        ):
+            message_label_stored = [
+                m
+                for (m,) in self._session.query(Labels.label_id)
+                .filter(Labels.user_id == user_id)
+                .filter(Labels.email_id == message_id)
+                .all()
+            ]
+            if message_label_stored == message_labels:
+                continue
+            else:
+                message_label_stored_set = set(message_label_stored)
+                message_labels_set = set(message_labels)
+                labels_to_add = list(
+                    message_labels_set.difference(message_label_stored_set)
+                )
+                labels_to_remove = list(
+                    message_label_stored_set.difference(message_labels_set)
+                )
+                if len(labels_to_add) > 0:
+                    self._session.add_all(
+                        [
+                            Labels(
+                                email_id=message_id,
+                                label_id=label_id,
+                                user_id=user_id,
+                            )
+                            for label_id in labels_to_add
+                        ]
+                    )
+                if len(labels_to_remove) > 0:
+                    _ = [
+                        self._session.query(Labels)
+                        .filter(Labels.user_id == user_id)
+                        .filter(Labels.email_id == message_id)
+                        .filter(Labels.label_id == label_id)
+                        .delete()
+                        for label_id in labels_to_remove
+                    ]
+                self._session.commit()
+
+    def get_all_emails(
+        self, include_deleted: bool = False, user_id: int = 1
+    ) -> pandas.DataFrame:
+        if include_deleted:
+            email_collect_lst = [
+                [
+                    email.email_id,
+                    email.email_subject,
+                    email.email_content,
+                    email.email_date,
+                ]
+                for email in self._session.query(EmailContent)
+                .filter(EmailContent.user_id == user_id)
+                .all()
+            ]
+        else:
+            email_collect_lst = [
+                [
+                    email.email_id,
+                    email.email_subject,
+                    email.email_content,
+                    email.email_date,
+                ]
+                for email in self._session.query(EmailContent)
+                .filter(EmailContent.user_id == user_id)
+                .filter(EmailContent.email_deleted.is_(False))
+                .all()
+            ]
+        return self._create_dataframe(
+            email_collect_lst=email_collect_lst,
+            user_id=user_id,
+            desc="Create dataframe from database",
+        )
+
+    def get_emails_by_label(
+        self, label_id: str, include_deleted: bool = False, user_id: int = 1
+    ) -> pandas.DataFrame:
+        return self.get_email_collection(
+            email_id_lst=[
+                email_id
+                for (email_id,) in self._session.query(Labels.email_id)
+                .filter(Labels.user_id == user_id)
+                .filter(Labels.label_id == label_id)
+                .all()
+            ],
+            include_deleted=include_deleted,
+            user_id=user_id,
+            desc="Create dataframe from emails by label",
+        )
+
+    def get_emails_by_from(
+        self, email_from: str, include_deleted: bool = False, user_id: int = 1
+    ) -> pandas.DataFrame:
+        return self.get_email_collection(
+            email_id_lst=[
+                email_id
+                for (email_id,) in self._session.query(EmailFrom.email_id)
+                .filter(EmailFrom.user_id == user_id)
+                .filter(EmailFrom.email_from == email_from)
+                .all()
+            ],
+            include_deleted=include_deleted,
+            user_id=user_id,
+            desc="Create dataframe from emails by from",
+        )
+
+    def get_emails_by_to(
+        self, email_to: str, include_deleted: bool = False, user_id: int = 1
+    ) -> pandas.DataFrame:
+        return self.get_email_collection(
+            email_id_lst=[
+                email_id
+                for (email_id,) in self._session.query(EmailTo.email_id)
+                .filter(EmailTo.user_id == user_id)
+                .filter(EmailTo.email_to == email_to)
+                .all()
+            ],
+            include_deleted=include_deleted,
+            user_id=user_id,
+            desc="Create dataframe from emails by to",
+        )
+
+    def get_emails_by_cc(
+        self, email_cc: str, include_deleted: bool = False, user_id: int = 1
+    ) -> pandas.DataFrame:
+        return self.get_email_collection(
+            email_id_lst=[
+                email_id
+                for (email_id,) in self._session.query(EmailTo.email_id)
+                .filter(EmailCc.user_id == user_id)
+                .filter(EmailCc.email_cc == email_cc)
+                .all()
+            ],
+            include_deleted=include_deleted,
+            user_id=user_id,
+            desc="Create dataframe from emails by cc",
+        )
+
+    def get_emails_by_thread(
+        self, thread_id: str, include_deleted: bool = False, user_id: int = 1
+    ) -> pandas.DataFrame:
+        return self.get_email_collection(
+            email_id_lst=[
+                email_id
+                for (email_id,) in self._session.query(Threads.email_id)
+                .filter(Threads.user_id == user_id)
+                .filter(Threads.thread_id == thread_id)
+                .all()
+            ],
+            include_deleted=include_deleted,
+            user_id=user_id,
+            desc="Create dataframe from emails by thread",
+        )
+
+    def get_email_collection(
+        self,
+        email_id_lst: list[str],
+        include_deleted: bool = False,
+        user_id: int = 1,
+        desc: str = "Create dataframe from email collection",
+    ) -> pandas.DataFrame:
+        if include_deleted:
+            email_collect_lst = [
+                [
+                    email.email_id,
+                    email.email_subject,
+                    email.email_content,
+                    email.email_date,
+                ]
+                for email in self._session.query(EmailContent)
+                .filter(EmailContent.user_id == user_id)
+                .filter(EmailContent.email_id.in_(email_id_lst))
+                .all()
+            ]
+        else:
+            email_collect_lst = [
+                [
+                    email.email_id,
+                    email.email_subject,
+                    email.email_content,
+                    email.email_date,
+                ]
+                for email in self._session.query(EmailContent)
+                .filter(EmailContent.user_id == user_id)
+                .filter(EmailContent.email_id.in_(email_id_lst))
+                .filter(EmailContent.email_deleted.is_(False))
+                .all()
+            ]
+        return self._create_dataframe(
+            email_collect_lst=email_collect_lst, user_id=user_id, desc=desc
+        )
+
+    def _commit_thread_table(self, df: pandas.DataFrame, user_id: int = 1) -> None:
+        self._session.add_all(
+            [
+                Threads(email_id=email_id, thread_id=thread_id, user_id=user_id)
+                for email_id, thread_id in zip(df["id"], df["threads"], strict=False)
+            ]
+        )
+        self._session.commit()
+
+    def _commit_email_from_table(self, df: pandas.DataFrame, user_id: int = 1) -> None:
+        self._session.add_all(
+            [
+                EmailFrom(email_id=email_id, email_from=email_from, user_id=user_id)
+                for email_id, email_from in zip(df["id"], df["from"], strict=False)
+            ]
+        )
+        self._session.commit()
+
+    def _commit_label_table(self, df: pandas.DataFrame, user_id: int = 1) -> None:
+        label_lst = []
+        for email_id, lid_lst in zip(df["id"], df["labels"], strict=False):
+            for label_id in lid_lst:
+                label_lst.append(
+                    Labels(email_id=email_id, label_id=label_id, user_id=user_id)
+                )
+        self._session.add_all(label_lst)
+        self._session.commit()
+
+    def _commit_email_to_table(self, df: pandas.DataFrame, user_id: int = 1) -> None:
+        email_to_lst = []
+        for email_id, email_lst in zip(df["id"], df["to"], strict=False):
+            for email_to in email_lst:
+                email_to_lst.append(
+                    EmailTo(email_id=email_id, email_to=email_to, user_id=user_id)
+                )
+        self._session.add_all(email_to_lst)
+        self._session.commit()
+
+    def _commit_email_cc_table(self, df: pandas.DataFrame, user_id: int = 1) -> None:
+        email_cc_lst = []
+        for email_id, email_lst in zip(df["id"], df["cc"], strict=False):
+            for email_cc in email_lst:
+                email_cc_lst.append(
+                    EmailCc(email_id=email_id, email_cc=email_cc, user_id=user_id)
+                )
+        self._session.add_all(email_cc_lst)
+        self._session.commit()
+
+    def _commit_content_table(self, df: pandas.DataFrame, user_id: int = 1) -> None:
+        self._session.add_all(
+            [
+                EmailContent(
+                    email_id=email_id,
+                    email_subject=email_subject,
+                    email_content=email_content,
+                    email_deleted=False,
+                    email_date=email_date,
+                    user_id=user_id,
+                )
+                for email_id, email_subject, email_content, email_date in zip(
+                    df["id"], df["subject"], df["content"], df["date"], strict=False
+                )
+            ]
+        )
+        self._session.commit()
+
+    def _create_dataframe(
+        self,
+        email_collect_lst: list[list[Any]],
+        user_id: int = 1,
+        desc: str = "Create dataframe from email list",
+    ) -> pandas.DataFrame:
+        (
+            email_id_lst,
+            email_subject_lst,
+            email_content_lst,
+            email_from_lst,
+            email_to_lst,
+            email_cc_lst,
+            email_threads_lst,
+            email_labels_lst,
+            email_date_lst,
+        ) = ([], [], [], [], [], [], [], [], [])
+        for email_id, email_subject, email_content, email_date in tqdm(
+            iterable=email_collect_lst, desc=desc
+        ):
+            email_from = [
+                email_from.email_from
+                for email_from in self._session.query(EmailFrom)
+                .filter(EmailFrom.user_id == user_id)
+                .filter(EmailFrom.email_id == email_id)
+                .all()
+            ]
+            email_to = [
+                email_to.email_to
+                for email_to in self._session.query(EmailTo)
+                .filter(EmailTo.user_id == user_id)
+                .filter(EmailTo.email_id == email_id)
+                .all()
+            ]
+            email_cc = [
+                email_cc.email_cc
+                for email_cc in self._session.query(EmailCc)
+                .filter(EmailCc.user_id == user_id)
+                .filter(EmailCc.email_id == email_id)
+                .all()
+            ]
+            label_lst = [
+                labels.label_id
+                for labels in self._session.query(Labels)
+                .filter(Labels.user_id == user_id)
+                .filter(Labels.email_id == email_id)
+                .all()
+            ]
+            thread_lst = [
+                threads.thread_id
+                for threads in self._session.query(Threads)
+                .filter(Threads.user_id == user_id)
+                .filter(Threads.email_id == email_id)
+                .all()
+            ]
+            if len(email_from) > 0:
+                email_from_lst.append(email_from[0])
+            else:
+                email_from_lst.append(None)
+            email_cc_lst.append(email_cc)
+            email_to_lst.append(email_to)
+            email_labels_lst.append(label_lst)
+            email_threads_lst.append(thread_lst[0])
+            email_id_lst.append(email_id)
+            email_subject_lst.append(email_subject)
+            email_content_lst.append(email_content)
+            email_date_lst.append(email_date)
+        return pandas.DataFrame(
+            {
+                "id": email_id_lst,
+                "from": email_from_lst,
+                "to": email_to_lst,
+                "cc": email_cc_lst,
+                "date": email_date_lst,
+                "threads": email_threads_lst,
+                "labels": email_labels_lst,
+                "subject": email_subject_lst,
+                "content": email_content_lst,
+            }
+        )
+
+
+def get_email_database(engine: Engine, session: Session) -> DatabaseInterface:
+    Base.metadata.create_all(engine)
+    return DatabaseInterface(session=session)