mailsort 0.0.3__tar.gz → 0.0.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. {mailsort-0.0.3 → mailsort-0.0.4}/PKG-INFO +20 -25
  2. {mailsort-0.0.3 → mailsort-0.0.4}/README.md +14 -23
  3. {mailsort-0.0.3 → mailsort-0.0.4}/pyproject.toml +8 -1
  4. {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/__main__.py +8 -94
  5. {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/_version.py +2 -2
  6. mailsort-0.0.4/src/mailsort/base/__init__.py +3 -0
  7. {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/base/database.py +85 -19
  8. {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/base/message.py +4 -0
  9. {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/evaluation.py +3 -4
  10. {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/imap/mail.py +2 -3
  11. {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/ml/database.py +79 -13
  12. {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/status.py +2 -3
  13. {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/training.py +2 -3
  14. mailsort-0.0.3/src/mailsort/base/__init__.py +0 -3
  15. {mailsort-0.0.3 → mailsort-0.0.4}/.gitignore +0 -0
  16. {mailsort-0.0.3 → mailsort-0.0.4}/LICENSE +0 -0
  17. {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/__init__.py +0 -0
  18. {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/api.py +0 -0
  19. {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/base/mail.py +0 -0
  20. {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/imap/__init__.py +0 -0
  21. {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/imap/authentication.py +0 -0
  22. {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/imap/message.py +0 -0
  23. {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/local.py +0 -0
  24. {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/ml/__init__.py +0 -0
  25. {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/ml/calibration.py +0 -0
  26. {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/ml/encoding.py +0 -0
  27. {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/ml/evaluation.py +0 -0
  28. {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/ml/model.py +0 -0
  29. {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/results.py +0 -0
  30. {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/sorter.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: mailsort
3
- Version: 0.0.3
3
+ Version: 0.0.4
4
4
  Summary: Assign labels to emails on any IMAP mail server based on their similarity to other emails assigned to the same label.
5
5
  Project-URL: Homepage, https://github.com/jan-janssen/mailsort
6
6
  Author-email: Jan Janssen <jan.janssen@outlook.com>
@@ -18,8 +18,12 @@ Requires-Python: >=3.10
18
18
  Requires-Dist: numpy<=2.5.1,>=1.23.5
19
19
  Requires-Dist: pandas<=3.0.5,>=1.5.3
20
20
  Requires-Dist: scikit-learn<=1.9.0,>=1.2.2
21
- Requires-Dist: sqlalchemy<=2.0.52,>=2.0.0
21
+ Requires-Dist: sqlalchemy<=2.0.54,>=2.0.0
22
22
  Requires-Dist: tqdm<=4.70.1,>=4.64.1
23
+ Provides-Extra: docs
24
+ Requires-Dist: myst-parser; extra == 'docs'
25
+ Requires-Dist: nbsphinx; extra == 'docs'
26
+ Requires-Dist: sphinx; extra == 'docs'
23
27
  Description-Content-Type: text/markdown
24
28
 
25
29
  # mailsort
@@ -28,14 +32,21 @@ Description-Content-Type: text/markdown
28
32
  [![Documentation Status](https://readthedocs.org/projects/mailsort/badge/?version=latest)](https://mailsort.readthedocs.io/en/latest/?badge=latest)
29
33
  [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
30
34
 
31
- Assign labels to emails on any IMAP mail server based on their similarity to other emails already
32
- assigned to the same label.
35
+ `mailsort` reduces the interruption caused by the increase of daily emails by automatically sorting your emails into
36
+ folders (IMAP labels) based on the similarity of new messages to the ones you have already sorted. It uses machine
37
+ learning to learn from your existing folder assignments and suggests or applies folders.
33
38
 
34
- `mailsort` connects to a mail account over plain IMAP, trains a machine learning model on the labels
35
- (IMAP folders) you have already assigned, and uses that model to suggest or apply labels to new
36
- messages. It has no dependency on Google APIs - for Gmail-specific features (OAuth, the Gmail label
37
- API, the sorting daemon and web UI) see [gmailsorter](https://github.com/jan-janssen/gmailsorter),
38
- which depends on `mailsort` for the shared IMAP and machine learning core.
39
+ This folder based approach allows you to focus on the most important emails first, while still being able to access less
40
+ important emails later. You are in contorl. When you change the folder assignments, you can retrain the `mailsort`
41
+ machine learning model so `mailsort` can adapt to your changing preferences. Think of it as a personal assistant that
42
+ helps you manage your emails.
43
+
44
+ `mailsort` is deliberately a plain Python library and command line tool rather than a hosted service: you run it
45
+ yourself, on your own schedule (for example from cron), against your own local database. Your emails belong to you.
46
+
47
+ `mailsort` connects to any IMAP mail server, stores your emails locally in an SQLite database, and trains a machine
48
+ learning model on the folders (IMAP labels) you have already assigned to your emails, and uses that model to suggest or
49
+ apply folders to new messages.
39
50
 
40
51
  To learn more about `mailsort` please have a look at the documentation below.
41
52
 
@@ -105,22 +116,6 @@ password manager.
105
116
 
106
117
  Run `mailsort --help` or `mailsort <command> --help` for the full list of options and examples.
107
118
 
108
- ### Upgrading from the pre-1.0 CLI
109
- The previous flat `-u/--update` and `-l/--label` options still work exactly as before, but are
110
- deprecated in favor of the subcommands above:
111
- ```
112
- mailsort --host imap.example.com --username user@example.com --password "..." -u
113
- mailsort --host imap.example.com --username user@example.com --password "..." -l "some_label"
114
- mailsort --host imap.example.com --username user@example.com --password "..." -l "some_label" --dry-run
115
- ```
116
- is equivalent to:
117
- ```
118
- mailsort sync --host imap.example.com --username user@example.com --password "..."
119
- mailsort train
120
- mailsort sort some_label --host imap.example.com --username user@example.com --password "..."
121
- mailsort predict some_label --host imap.example.com --username user@example.com --password "..."
122
- ```
123
-
124
119
  ## Python interface
125
120
  The recommended way to use `mailsort` from Python is `MailSorter`, a small facade that wraps a
126
121
  mailbox backend (such as `Imap`) and exposes the fetch-store-train-predict-move loop through the
@@ -4,14 +4,21 @@
4
4
  [![Documentation Status](https://readthedocs.org/projects/mailsort/badge/?version=latest)](https://mailsort.readthedocs.io/en/latest/?badge=latest)
5
5
  [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
6
6
 
7
- Assign labels to emails on any IMAP mail server based on their similarity to other emails already
8
- assigned to the same label.
7
+ `mailsort` reduces the interruption caused by the increase of daily emails by automatically sorting your emails into
8
+ folders (IMAP labels) based on the similarity of new messages to the ones you have already sorted. It uses machine
9
+ learning to learn from your existing folder assignments and suggests or applies folders.
9
10
 
10
- `mailsort` connects to a mail account over plain IMAP, trains a machine learning model on the labels
11
- (IMAP folders) you have already assigned, and uses that model to suggest or apply labels to new
12
- messages. It has no dependency on Google APIs - for Gmail-specific features (OAuth, the Gmail label
13
- API, the sorting daemon and web UI) see [gmailsorter](https://github.com/jan-janssen/gmailsorter),
14
- which depends on `mailsort` for the shared IMAP and machine learning core.
11
+ This folder based approach allows you to focus on the most important emails first, while still being able to access less
12
+ important emails later. You are in contorl. When you change the folder assignments, you can retrain the `mailsort`
13
+ machine learning model so `mailsort` can adapt to your changing preferences. Think of it as a personal assistant that
14
+ helps you manage your emails.
15
+
16
+ `mailsort` is deliberately a plain Python library and command line tool rather than a hosted service: you run it
17
+ yourself, on your own schedule (for example from cron), against your own local database. Your emails belong to you.
18
+
19
+ `mailsort` connects to any IMAP mail server, stores your emails locally in an SQLite database, and trains a machine
20
+ learning model on the folders (IMAP labels) you have already assigned to your emails, and uses that model to suggest or
21
+ apply folders to new messages.
15
22
 
16
23
  To learn more about `mailsort` please have a look at the documentation below.
17
24
 
@@ -81,22 +88,6 @@ password manager.
81
88
 
82
89
  Run `mailsort --help` or `mailsort <command> --help` for the full list of options and examples.
83
90
 
84
- ### Upgrading from the pre-1.0 CLI
85
- The previous flat `-u/--update` and `-l/--label` options still work exactly as before, but are
86
- deprecated in favor of the subcommands above:
87
- ```
88
- mailsort --host imap.example.com --username user@example.com --password "..." -u
89
- mailsort --host imap.example.com --username user@example.com --password "..." -l "some_label"
90
- mailsort --host imap.example.com --username user@example.com --password "..." -l "some_label" --dry-run
91
- ```
92
- is equivalent to:
93
- ```
94
- mailsort sync --host imap.example.com --username user@example.com --password "..."
95
- mailsort train
96
- mailsort sort some_label --host imap.example.com --username user@example.com --password "..."
97
- mailsort predict some_label --host imap.example.com --username user@example.com --password "..."
98
- ```
99
-
100
91
  ## Python interface
101
92
  The recommended way to use `mailsort` from Python is `MailSorter`, a small facade that wraps a
102
93
  mailbox backend (such as `Imap`) and exposes the fetch-store-train-predict-move loop through the
@@ -27,7 +27,7 @@ dependencies = [
27
27
  "tqdm>=4.64.1,<=4.70.1",
28
28
  "pandas>=1.5.3,<=3.0.5",
29
29
  "scikit-learn>=1.2.2,<=1.9.0",
30
- "sqlalchemy>=2.0.0,<=2.0.52",
30
+ "sqlalchemy>=2.0.0,<=2.0.54",
31
31
  ]
32
32
 
33
33
  [project.urls]
@@ -36,6 +36,13 @@ Homepage = "https://github.com/jan-janssen/mailsort"
36
36
  [project.scripts]
37
37
  mailsort = "mailsort.__main__:command_line_parser"
38
38
 
39
+ [project.optional-dependencies]
40
+ docs = [
41
+ "nbsphinx",
42
+ "sphinx",
43
+ "myst-parser",
44
+ ]
45
+
39
46
  [tool.ruff.lint]
40
47
  select = [
41
48
  # pycodestyle
@@ -263,31 +263,12 @@ Run `mailsort <command> --help` for the arguments of an individual command.
263
263
  """,
264
264
  formatter_class=argparse.RawDescriptionHelpFormatter,
265
265
  )
266
- # Top-level flags kept for backwards compatibility with the pre-subcommand CLI
267
- # (deprecated - prefer the sync/train/predict/sort/status subcommands below).
266
+ # Top-level flags so they can be given before the subcommand, e.g. "mailsort --host x
267
+ # sync" - see _add_connection_arguments for why the subcommand copies suppress their
268
+ # defaults.
268
269
  _add_connection_arguments(parser)
269
270
  _add_database_argument(parser)
270
271
  _add_identification_argument(parser)
271
- parser.add_argument(
272
- "-u",
273
- "--update",
274
- action="store_true",
275
- help="Deprecated - use 'mailsort sync' followed by 'mailsort train' instead. "
276
- "Update local database and retrain machine learning model.",
277
- )
278
- parser.add_argument(
279
- "-l",
280
- "--label",
281
- help="Deprecated - use 'mailsort predict FOLDER' or 'mailsort sort FOLDER' instead. "
282
- "Email label (IMAP folder) to be filtered with machine learning.",
283
- )
284
- parser.add_argument(
285
- "-n",
286
- "--dry-run",
287
- action="store_true",
288
- help="Deprecated - use 'mailsort predict FOLDER' instead. With -l/--label, print "
289
- "the machine learning recommendations for that folder instead of moving any messages.",
290
- )
291
272
 
292
273
  subparsers = parser.add_subparsers(
293
274
  dest="command", metavar="{sync,train,predict,sort,status,evaluate}"
@@ -557,73 +538,6 @@ def _run_evaluate(args, database, db_user_id):
557
538
  return _EXIT_OK
558
539
 
559
540
 
560
- def _run_legacy(args, parser, database, db_user_id):
561
- """
562
- Pre-subcommand CLI, kept for backwards compatibility with existing scripts using
563
- -u/--update or -l/--label - see sync/train/predict/sort for the replacements.
564
- """
565
- if not args.host or not args.username:
566
- print("Please provide --host and --username.")
567
- return _EXIT_CONFIG_ERROR
568
- if not args.password:
569
- print("Please provide --password.")
570
- return _EXIT_CONFIG_ERROR
571
- if args.update:
572
- print(
573
- "mailsort: -u/--update is deprecated, use 'mailsort sync' followed by "
574
- "'mailsort train' instead.",
575
- file=sys.stderr,
576
- )
577
- imap = _connect(args, database, db_user_id)
578
- try:
579
- imap.update_database(quick=False)
580
- imap.fit_machine_learning_model_to_database(
581
- n_estimators=100,
582
- max_features=400,
583
- random_state=42,
584
- bootstrap=True,
585
- include_deleted=False,
586
- )
587
- finally:
588
- imap.close()
589
- return _EXIT_OK
590
- elif args.label and args.dry_run:
591
- print(
592
- "mailsort: -l/--label with -n/--dry-run is deprecated, use "
593
- "'mailsort predict FOLDER' instead.",
594
- file=sys.stderr,
595
- )
596
- imap = _connect(args, database, db_user_id)
597
- try:
598
- recommendations = imap.get_label_recommendations(
599
- label=args.label,
600
- recommendation_ratio=_DEFAULT_RECOMMENDATION_RATIO,
601
- label_prefix=_DEFAULT_LABEL_PREFIX,
602
- )
603
- finally:
604
- imap.close()
605
- print(_format_recommendations_table(recommendations))
606
- return _EXIT_OK
607
- elif args.label:
608
- print(
609
- "mailsort: -l/--label is deprecated, use 'mailsort sort FOLDER' instead.",
610
- file=sys.stderr,
611
- )
612
- imap = _connect(args, database, db_user_id)
613
- try:
614
- imap.filter_messages_from_server(
615
- label=args.label,
616
- recommendation_ratio=_DEFAULT_RECOMMENDATION_RATIO,
617
- label_prefix=_DEFAULT_LABEL_PREFIX,
618
- )
619
- finally:
620
- imap.close()
621
- return _EXIT_OK
622
- else:
623
- parser.print_help()
624
- return _EXIT_OK
625
-
626
-
627
541
  _COMMAND_HANDLERS = {
628
542
  "sync": _run_sync,
629
543
  "train": _run_train,
@@ -657,12 +571,12 @@ def command_line_parser(argv=None):
657
571
  database = getattr(args, "database", None) or _DEFAULT_DATABASE
658
572
  handler = _COMMAND_HANDLERS.get(args.command)
659
573
 
574
+ if handler is None:
575
+ parser.print_help()
576
+ return _EXIT_OK
577
+
660
578
  try:
661
- if handler is not None:
662
- return handler(args=args, database=database, db_user_id=db_user_id)
663
- return _run_legacy(
664
- args=args, parser=parser, database=database, db_user_id=db_user_id
665
- )
579
+ return handler(args=args, database=database, db_user_id=db_user_id)
666
580
  except Exception as error:
667
581
  print(f"mailsort: error: {error}", file=sys.stderr)
668
582
  return _EXIT_RUNTIME_ERROR
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
18
18
  commit_id: str | None
19
19
  __commit_id__: str | None
20
20
 
21
- __version__ = version = '0.0.3'
22
- __version_tuple__ = version_tuple = (0, 0, 3)
21
+ __version__ = version = '0.0.4'
22
+ __version_tuple__ = version_tuple = (0, 0, 4)
23
23
 
24
24
  __commit_id__ = commit_id = None
@@ -0,0 +1,3 @@
1
+ from mailsort.base.database import create_database_engine, get_email_database
2
+
3
+ __all__ = ["create_database_engine", "get_email_database"]
@@ -1,19 +1,72 @@
1
1
  from typing import Any
2
2
 
3
3
  import pandas
4
- from sqlalchemy import Boolean, Column, DateTime, Engine, ForeignKey, Integer, String
4
+ from sqlalchemy import (
5
+ URL,
6
+ Boolean,
7
+ Column,
8
+ DateTime,
9
+ Engine,
10
+ ForeignKey,
11
+ Integer,
12
+ String,
13
+ Text,
14
+ create_engine,
15
+ make_url,
16
+ )
17
+ from sqlalchemy.dialects.mysql import LONGTEXT
5
18
  from sqlalchemy.orm import Session, declarative_base
6
19
  from tqdm import tqdm
7
20
 
8
21
  Base = declarative_base()
9
22
 
23
+ # MySQL requires a length for VARCHAR columns. Identifiers, labels and single
24
+ # email addresses fit in VARCHAR(255), while subjects and recipient lists can
25
+ # exceed 255 characters and are stored as TEXT. The email content can be larger
26
+ # than the 64 KB limit of TEXT, so it is stored as LONGTEXT on MySQL. Since
27
+ # MySQL 8.4 a foreign key has to reference a unique key, so email_id is unique.
28
+ SHORT_STRING_LENGTH = 255
29
+ LongText = Text().with_variant(LONGTEXT(), "mysql")
30
+
31
+ # MySQL's utf8 charset only supports characters of up to 3 bytes, so emojis in
32
+ # subjects or content require utf8mb4. The binary collation keeps comparisons
33
+ # case-sensitive, like in SQLite.
34
+ MYSQL_TABLE_ARGS = {"mysql_charset": "utf8mb4", "mysql_collate": "utf8mb4_bin"}
35
+ _MYSQL_CHARSET_DRIVERS = ("mysqldb", "pymysql")
36
+
37
+
38
+ def _with_mysql_charset(connection_str: str | URL) -> URL:
39
+ """
40
+ Add charset=utf8mb4 to MySQL connection strings without an explicit charset,
41
+ so 4 byte characters are also transmitted correctly.
42
+ """
43
+ url = make_url(connection_str)
44
+ if url.get_driver_name() in _MYSQL_CHARSET_DRIVERS and "charset" not in url.query:
45
+ url = url.update_query_dict({"charset": "utf8mb4"})
46
+ return url
47
+
48
+
49
+ def create_database_engine(connection_str: str | URL) -> Engine:
50
+ return create_engine(_with_mysql_charset(connection_str))
51
+
52
+
53
+ def _missing_to_none(value: Any) -> Any:
54
+ """
55
+ Convert missing values in a pandas.DataFrame, like NaN or NaT, to None, as
56
+ MySQL rejects NaN while None is stored as NULL.
57
+ """
58
+ if pandas.api.types.is_scalar(value) and pandas.isna(value):
59
+ return None
60
+ return value
61
+
10
62
 
11
63
  class EmailContent(Base):
12
64
  __tablename__ = "email_content"
65
+ __table_args__ = MYSQL_TABLE_ARGS
13
66
  id = Column(Integer, primary_key=True)
14
- email_id = Column(String)
15
- email_subject = Column(String)
16
- email_content = Column(String)
67
+ email_id = Column(String(SHORT_STRING_LENGTH), unique=True)
68
+ email_subject = Column(Text)
69
+ email_content = Column(LongText)
17
70
  email_deleted = Column(Boolean)
18
71
  email_date = Column(DateTime)
19
72
  user_id = Column(Integer)
@@ -21,41 +74,46 @@ class EmailContent(Base):
21
74
 
22
75
  class Threads(Base):
23
76
  __tablename__ = "email_threads"
77
+ __table_args__ = MYSQL_TABLE_ARGS
24
78
  id = Column(Integer, primary_key=True)
25
- email_id = Column(String, ForeignKey("email_content.email_id"))
26
- thread_id = Column(String)
79
+ email_id = Column(String(SHORT_STRING_LENGTH), ForeignKey("email_content.email_id"))
80
+ thread_id = Column(String(SHORT_STRING_LENGTH))
27
81
  user_id = Column(Integer)
28
82
 
29
83
 
30
84
  class Labels(Base):
31
85
  __tablename__ = "email_labels"
86
+ __table_args__ = MYSQL_TABLE_ARGS
32
87
  id = Column(Integer, primary_key=True)
33
- email_id = Column(String, ForeignKey("email_content.email_id"))
34
- label_id = Column(String)
88
+ email_id = Column(String(SHORT_STRING_LENGTH), ForeignKey("email_content.email_id"))
89
+ label_id = Column(String(SHORT_STRING_LENGTH))
35
90
  user_id = Column(Integer)
36
91
 
37
92
 
38
93
  class EmailTo(Base):
39
94
  __tablename__ = "email_to"
95
+ __table_args__ = MYSQL_TABLE_ARGS
40
96
  id = Column(Integer, primary_key=True)
41
- email_id = Column(String, ForeignKey("email_content.email_id"))
42
- email_to = Column(String)
97
+ email_id = Column(String(SHORT_STRING_LENGTH), ForeignKey("email_content.email_id"))
98
+ email_to = Column(Text)
43
99
  user_id = Column(Integer)
44
100
 
45
101
 
46
102
  class EmailCc(Base):
47
103
  __tablename__ = "email_cc"
104
+ __table_args__ = MYSQL_TABLE_ARGS
48
105
  id = Column(Integer, primary_key=True)
49
- email_id = Column(String, ForeignKey("email_content.email_id"))
50
- email_cc = Column(String)
106
+ email_id = Column(String(SHORT_STRING_LENGTH), ForeignKey("email_content.email_id"))
107
+ email_cc = Column(Text)
51
108
  user_id = Column(Integer)
52
109
 
53
110
 
54
111
  class EmailFrom(Base):
55
112
  __tablename__ = "email_from"
113
+ __table_args__ = MYSQL_TABLE_ARGS
56
114
  id = Column(Integer, primary_key=True)
57
- email_id = Column(String, ForeignKey("email_content.email_id"))
58
- email_from = Column(String)
115
+ email_id = Column(String(SHORT_STRING_LENGTH), ForeignKey("email_content.email_id"))
116
+ email_from = Column(String(SHORT_STRING_LENGTH))
59
117
  user_id = Column(Integer)
60
118
 
61
119
 
@@ -333,7 +391,11 @@ class DatabaseInterface(DatabaseTemplate):
333
391
  def _commit_thread_table(self, df: pandas.DataFrame, user_id: int = 1) -> None:
334
392
  self._session.add_all(
335
393
  [
336
- Threads(email_id=email_id, thread_id=thread_id, user_id=user_id)
394
+ Threads(
395
+ email_id=email_id,
396
+ thread_id=_missing_to_none(thread_id),
397
+ user_id=user_id,
398
+ )
337
399
  for email_id, thread_id in zip(df["id"], df["threads"], strict=False)
338
400
  ]
339
401
  )
@@ -342,7 +404,11 @@ class DatabaseInterface(DatabaseTemplate):
342
404
  def _commit_email_from_table(self, df: pandas.DataFrame, user_id: int = 1) -> None:
343
405
  self._session.add_all(
344
406
  [
345
- EmailFrom(email_id=email_id, email_from=email_from, user_id=user_id)
407
+ EmailFrom(
408
+ email_id=email_id,
409
+ email_from=_missing_to_none(email_from),
410
+ user_id=user_id,
411
+ )
346
412
  for email_id, email_from in zip(df["id"], df["from"], strict=False)
347
413
  ]
348
414
  )
@@ -383,10 +449,10 @@ class DatabaseInterface(DatabaseTemplate):
383
449
  [
384
450
  EmailContent(
385
451
  email_id=email_id,
386
- email_subject=email_subject,
387
- email_content=email_content,
452
+ email_subject=_missing_to_none(email_subject),
453
+ email_content=_missing_to_none(email_content),
388
454
  email_deleted=False,
389
- email_date=email_date,
455
+ email_date=_missing_to_none(email_date),
390
456
  user_id=user_id,
391
457
  )
392
458
  for email_id, email_subject, email_content, email_date in zip(
@@ -6,6 +6,7 @@ from typing import Any
6
6
 
7
7
  _MAX_DATE_COMMAS = 2
8
8
  _DATE_HYPHEN_COUNT = 2
9
+ _WEEKDAY_ABBREVIATION_LENGTH = 3
9
10
 
10
11
 
11
12
  # https://stackoverflow.com/questions/753052/strip-html-from-strings-in-python
@@ -37,6 +38,9 @@ def email_date_converter(email_date: Any) -> datetime | None:
37
38
  email_date = email_date.replace("\xa0", "")
38
39
  if email_date.count(",") >= _MAX_DATE_COMMAS:
39
40
  email_date = ", ".join(email_date.split(", ")[-2:])
41
+ weekday, separator, remainder = email_date.partition(", ")
42
+ if separator and weekday.isalpha() and len(weekday) != _WEEKDAY_ABBREVIATION_LENGTH:
43
+ email_date = remainder
40
44
  if email_date[-3:-2].isalpha():
41
45
  email_date = " ".join(email_date.split()[:-1])
42
46
  if email_date[-1].isalpha():
@@ -8,10 +8,9 @@ a mail backend such as mailsort.Imap, which would require live mail server crede
8
8
  not need.
9
9
  """
10
10
 
11
- from sqlalchemy import create_engine
12
11
  from sqlalchemy.orm import sessionmaker
13
12
 
14
- from mailsort.base import get_email_database
13
+ from mailsort.base import create_database_engine, get_email_database
15
14
  from mailsort.ml.evaluation import (
16
15
  EvaluationReport,
17
16
  FolderMetrics,
@@ -76,7 +75,7 @@ def evaluate_models(
76
75
  Returns:
77
76
  mailsort.ml.evaluation.EvaluationReport
78
77
  """
79
- engine = create_engine(connection_str)
78
+ engine = create_database_engine(connection_str)
80
79
  session = sessionmaker(bind=engine)()
81
80
  try:
82
81
  db_email = get_email_database(engine=engine, session=session)
@@ -131,7 +130,7 @@ def score_holdout(
131
130
  Returns:
132
131
  mailsort.ml.evaluation.HoldoutScores
133
132
  """
134
- engine = create_engine(connection_str)
133
+ engine = create_database_engine(connection_str)
135
134
  session = sessionmaker(bind=engine)()
136
135
  try:
137
136
  db_email = get_email_database(engine=engine, session=session)
@@ -3,10 +3,9 @@ import email
3
3
  import imaplib
4
4
  import re
5
5
 
6
- from sqlalchemy import create_engine
7
6
  from sqlalchemy.orm import sessionmaker
8
7
 
9
- from mailsort.base import get_email_database
8
+ from mailsort.base import create_database_engine, get_email_database
10
9
  from mailsort.base.mail import AbstractMailBox
11
10
  from mailsort.imap.message import get_email_dict
12
11
  from mailsort.ml import get_machine_learning_database
@@ -341,7 +340,7 @@ class ImapMailBase(AbstractMailBox):
341
340
 
342
341
  @staticmethod
343
342
  def _create_databases(connection_str):
344
- engine = create_engine(connection_str)
343
+ engine = create_database_engine(connection_str)
345
344
  session = sessionmaker(bind=engine)()
346
345
  db_email = get_email_database(engine=engine, session=session)
347
346
  db_ml = get_machine_learning_database(engine=engine, session=session)
@@ -1,26 +1,53 @@
1
1
  import pickle
2
+ import zlib
2
3
 
3
4
  from sklearn.ensemble import RandomForestClassifier
4
- from sqlalchemy import Column, Engine, Integer, String
5
+ from sqlalchemy import Column, Engine, Integer, LargeBinary, String, Text
6
+ from sqlalchemy.dialects.mysql import LONGBLOB
5
7
  from sqlalchemy.orm import Session, declarative_base
6
8
 
7
- from mailsort.base.database import DatabaseTemplate
9
+ from mailsort.base.database import (
10
+ MYSQL_TABLE_ARGS,
11
+ SHORT_STRING_LENGTH,
12
+ DatabaseTemplate,
13
+ )
8
14
 
9
15
  Base = declarative_base()
10
16
 
17
+ # Pickled random forest models easily exceed the 16 MB limit of MEDIUMBLOB
18
+ LongBinary = LargeBinary().with_variant(LONGBLOB(), "mysql")
19
+
20
+ # MySQL rejects statements larger than max_allowed_packet (64 MB by default and
21
+ # at most 1 GB), while a pickled calibrated random forest can be several hundred
22
+ # MB. So the compressed pickle is split into chunks, stored as individual rows.
23
+ MODEL_CHUNK_SIZE = 1024 * 1024
24
+
11
25
 
12
26
  class MachineLearningLabels(Base):
13
27
  __tablename__ = "ml_labels"
28
+ __table_args__ = MYSQL_TABLE_ARGS
29
+ id = Column(Integer, primary_key=True)
30
+ label_id = Column(String(SHORT_STRING_LENGTH))
31
+ random_forest = Column(LongBinary)
32
+ user_id = Column(Integer)
33
+
34
+
35
+ class MachineLearningModelChunks(Base):
36
+ __tablename__ = "ml_label_chunks"
37
+ __table_args__ = MYSQL_TABLE_ARGS
14
38
  id = Column(Integer, primary_key=True)
15
- label_id = Column(String)
16
- random_forest = Column(String)
39
+ label_id = Column(String(SHORT_STRING_LENGTH))
40
+ chunk_index = Column(Integer)
41
+ data = Column(LongBinary)
17
42
  user_id = Column(Integer)
18
43
 
19
44
 
20
45
  class MachineLearningFeatures(Base):
21
46
  __tablename__ = "ml_features"
47
+ __table_args__ = MYSQL_TABLE_ARGS
22
48
  id = Column(Integer, primary_key=True)
23
- feature = Column(String)
49
+ # Features are derived from the recipients, which can exceed 255 characters
50
+ feature = Column(Text)
24
51
  user_id = Column(Integer)
25
52
 
26
53
 
@@ -68,12 +95,12 @@ class MachineLearningDatabase(DatabaseTemplate):
68
95
  if len(model_dict_new) > 0:
69
96
  self._session.add_all(
70
97
  [
71
- MachineLearningLabels(
72
- label_id=k, random_forest=pickle.dumps(v), user_id=user_id
73
- )
74
- for k, v in model_dict_new.items()
98
+ MachineLearningLabels(label_id=k, user_id=user_id)
99
+ for k in model_dict_new
75
100
  ]
76
101
  )
102
+ for k, v in model_dict_new.items():
103
+ self._store_model_chunks(label=k, model=v, user_id=user_id)
77
104
  if len(feature_new_lst) > 0:
78
105
  self._session.add_all(
79
106
  [
@@ -91,13 +118,17 @@ class MachineLearningDatabase(DatabaseTemplate):
91
118
  .all()
92
119
  )
93
120
  for label_obj in label_obj_lst:
94
- label_obj.random_forest = pickle.dumps(
95
- model_dict_update[label_obj.label_id]
96
- )
121
+ label_obj.random_forest = None
122
+ self._delete_model_chunks(
123
+ label_lst=list(model_dict_update.keys()), user_id=user_id
124
+ )
125
+ for k, v in model_dict_update.items():
126
+ self._store_model_chunks(label=k, model=v, user_id=user_id)
97
127
  if len(model_delete_lst) > 0:
98
128
  self._session.query(MachineLearningLabels).filter(
99
129
  MachineLearningLabels.user_id == user_id
100
130
  ).filter(MachineLearningLabels.label_id.in_(model_delete_lst)).delete()
131
+ self._delete_model_chunks(label_lst=model_delete_lst, user_id=user_id)
101
132
  if len(feature_remove_lst) > 0:
102
133
  self._session.query(MachineLearningFeatures).filter(
103
134
  MachineLearningFeatures.user_id == user_id
@@ -124,10 +155,45 @@ class MachineLearningDatabase(DatabaseTemplate):
124
155
  )
125
156
  feature_lst = self.get_features(user_id=user_id)
126
157
  return {
127
- label_obj.label_id: pickle.loads(label_obj.random_forest)
158
+ label_obj.label_id: (
159
+ # Models stored before the chunked storage was introduced
160
+ pickle.loads(label_obj.random_forest)
161
+ if label_obj.random_forest is not None
162
+ else self._load_model_chunks(label=label_obj.label_id, user_id=user_id)
163
+ )
128
164
  for label_obj in label_obj_lst
129
165
  }, feature_lst
130
166
 
167
+ def _store_model_chunks(self, label: str, model, user_id: int = 1) -> None:
168
+ data = zlib.compress(pickle.dumps(model))
169
+ for chunk_index, start in enumerate(range(0, len(data), MODEL_CHUNK_SIZE)):
170
+ self._session.add(
171
+ MachineLearningModelChunks(
172
+ label_id=label,
173
+ chunk_index=chunk_index,
174
+ data=data[start : start + MODEL_CHUNK_SIZE],
175
+ user_id=user_id,
176
+ )
177
+ )
178
+ # Flush each chunk individually, so the chunks are not combined in
179
+ # a single multi-row INSERT statement exceeding max_allowed_packet
180
+ self._session.flush()
181
+
182
+ def _load_model_chunks(self, label: str, user_id: int = 1):
183
+ chunk_lst = (
184
+ self._session.query(MachineLearningModelChunks.data)
185
+ .filter(MachineLearningModelChunks.user_id == user_id)
186
+ .filter(MachineLearningModelChunks.label_id == label)
187
+ .order_by(MachineLearningModelChunks.chunk_index)
188
+ .all()
189
+ )
190
+ return pickle.loads(zlib.decompress(b"".join(chunk[0] for chunk in chunk_lst)))
191
+
192
+ def _delete_model_chunks(self, label_lst: list[str], user_id: int = 1) -> None:
193
+ self._session.query(MachineLearningModelChunks).filter(
194
+ MachineLearningModelChunks.user_id == user_id
195
+ ).filter(MachineLearningModelChunks.label_id.in_(label_lst)).delete()
196
+
131
197
  def get_labels(self, user_id: int = 1) -> list[str]:
132
198
  """
133
199
  List the folders/labels a machine learning model has been trained for.
@@ -8,10 +8,9 @@ connection to the mail server.
8
8
 
9
9
  from dataclasses import dataclass
10
10
 
11
- from sqlalchemy import create_engine
12
11
  from sqlalchemy.orm import sessionmaker
13
12
 
14
- from mailsort.base import get_email_database
13
+ from mailsort.base import create_database_engine, get_email_database
15
14
  from mailsort.ml import get_machine_learning_database
16
15
 
17
16
 
@@ -39,7 +38,7 @@ def get_database_status(connection_str, db_user_id=1):
39
38
  Returns:
40
39
  DatabaseStatus
41
40
  """
42
- engine = create_engine(connection_str)
41
+ engine = create_database_engine(connection_str)
43
42
  session = sessionmaker(bind=engine)()
44
43
  try:
45
44
  db_email = get_email_database(engine=engine, session=session)
@@ -8,10 +8,9 @@ instead of going through a mail backend such as mailsort.Imap, which would requi
8
8
  server credentials it does not need.
9
9
  """
10
10
 
11
- from sqlalchemy import create_engine
12
11
  from sqlalchemy.orm import sessionmaker
13
12
 
14
- from mailsort.base import get_email_database
13
+ from mailsort.base import create_database_engine, get_email_database
15
14
  from mailsort.ml import (
16
15
  DEFAULT_MAX_CALIBRATION_CV_FOLDS,
17
16
  DEFAULT_MIN_SAMPLES_PER_CLASS_FOR_CALIBRATION,
@@ -58,7 +57,7 @@ def train_machine_learning_models(
58
57
  Returns:
59
58
  int: number of per-folder models trained
60
59
  """
61
- engine = create_engine(connection_str)
60
+ engine = create_database_engine(connection_str)
62
61
  session = sessionmaker(bind=engine)()
63
62
  try:
64
63
  db_email = get_email_database(engine=engine, session=session)
@@ -1,3 +0,0 @@
1
- from mailsort.base.database import get_email_database
2
-
3
- __all__ = ["get_email_database"]
File without changes
File without changes
File without changes
File without changes