mailsort 0.0.3__tar.gz → 0.0.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mailsort-0.0.3 → mailsort-0.0.4}/PKG-INFO +20 -25
- {mailsort-0.0.3 → mailsort-0.0.4}/README.md +14 -23
- {mailsort-0.0.3 → mailsort-0.0.4}/pyproject.toml +8 -1
- {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/__main__.py +8 -94
- {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/_version.py +2 -2
- mailsort-0.0.4/src/mailsort/base/__init__.py +3 -0
- {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/base/database.py +85 -19
- {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/base/message.py +4 -0
- {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/evaluation.py +3 -4
- {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/imap/mail.py +2 -3
- {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/ml/database.py +79 -13
- {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/status.py +2 -3
- {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/training.py +2 -3
- mailsort-0.0.3/src/mailsort/base/__init__.py +0 -3
- {mailsort-0.0.3 → mailsort-0.0.4}/.gitignore +0 -0
- {mailsort-0.0.3 → mailsort-0.0.4}/LICENSE +0 -0
- {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/__init__.py +0 -0
- {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/api.py +0 -0
- {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/base/mail.py +0 -0
- {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/imap/__init__.py +0 -0
- {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/imap/authentication.py +0 -0
- {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/imap/message.py +0 -0
- {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/local.py +0 -0
- {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/ml/__init__.py +0 -0
- {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/ml/calibration.py +0 -0
- {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/ml/encoding.py +0 -0
- {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/ml/evaluation.py +0 -0
- {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/ml/model.py +0 -0
- {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/results.py +0 -0
- {mailsort-0.0.3 → mailsort-0.0.4}/src/mailsort/sorter.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: mailsort
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.4
|
|
4
4
|
Summary: Assign labels to emails on any IMAP mail server based on their similarity to other emails assigned to the same label.
|
|
5
5
|
Project-URL: Homepage, https://github.com/jan-janssen/mailsort
|
|
6
6
|
Author-email: Jan Janssen <jan.janssen@outlook.com>
|
|
@@ -18,8 +18,12 @@ Requires-Python: >=3.10
|
|
|
18
18
|
Requires-Dist: numpy<=2.5.1,>=1.23.5
|
|
19
19
|
Requires-Dist: pandas<=3.0.5,>=1.5.3
|
|
20
20
|
Requires-Dist: scikit-learn<=1.9.0,>=1.2.2
|
|
21
|
-
Requires-Dist: sqlalchemy<=2.0.
|
|
21
|
+
Requires-Dist: sqlalchemy<=2.0.54,>=2.0.0
|
|
22
22
|
Requires-Dist: tqdm<=4.70.1,>=4.64.1
|
|
23
|
+
Provides-Extra: docs
|
|
24
|
+
Requires-Dist: myst-parser; extra == 'docs'
|
|
25
|
+
Requires-Dist: nbsphinx; extra == 'docs'
|
|
26
|
+
Requires-Dist: sphinx; extra == 'docs'
|
|
23
27
|
Description-Content-Type: text/markdown
|
|
24
28
|
|
|
25
29
|
# mailsort
|
|
@@ -28,14 +32,21 @@ Description-Content-Type: text/markdown
|
|
|
28
32
|
[](https://mailsort.readthedocs.io/en/latest/?badge=latest)
|
|
29
33
|
[](https://github.com/psf/black)
|
|
30
34
|
|
|
31
|
-
|
|
32
|
-
|
|
35
|
+
`mailsort` reduces the interruption caused by the increase of daily emails by automatically sorting your emails into
|
|
36
|
+
folders (IMAP labels) based on the similarity of new messages to the ones you have already sorted. It uses machine
|
|
37
|
+
learning to learn from your existing folder assignments and suggests or applies folders.
|
|
33
38
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
+
This folder based approach allows you to focus on the most important emails first, while still being able to access less
|
|
40
|
+
important emails later. You are in contorl. When you change the folder assignments, you can retrain the `mailsort`
|
|
41
|
+
machine learning model so `mailsort` can adapt to your changing preferences. Think of it as a personal assistant that
|
|
42
|
+
helps you manage your emails.
|
|
43
|
+
|
|
44
|
+
`mailsort` is deliberately a plain Python library and command line tool rather than a hosted service: you run it
|
|
45
|
+
yourself, on your own schedule (for example from cron), against your own local database. Your emails belong to you.
|
|
46
|
+
|
|
47
|
+
`mailsort` connects to any IMAP mail server, stores your emails locally in an SQLite database, and trains a machine
|
|
48
|
+
learning model on the folders (IMAP labels) you have already assigned to your emails, and uses that model to suggest or
|
|
49
|
+
apply folders to new messages.
|
|
39
50
|
|
|
40
51
|
To learn more about `mailsort` please have a look at the documentation below.
|
|
41
52
|
|
|
@@ -105,22 +116,6 @@ password manager.
|
|
|
105
116
|
|
|
106
117
|
Run `mailsort --help` or `mailsort <command> --help` for the full list of options and examples.
|
|
107
118
|
|
|
108
|
-
### Upgrading from the pre-1.0 CLI
|
|
109
|
-
The previous flat `-u/--update` and `-l/--label` options still work exactly as before, but are
|
|
110
|
-
deprecated in favor of the subcommands above:
|
|
111
|
-
```
|
|
112
|
-
mailsort --host imap.example.com --username user@example.com --password "..." -u
|
|
113
|
-
mailsort --host imap.example.com --username user@example.com --password "..." -l "some_label"
|
|
114
|
-
mailsort --host imap.example.com --username user@example.com --password "..." -l "some_label" --dry-run
|
|
115
|
-
```
|
|
116
|
-
is equivalent to:
|
|
117
|
-
```
|
|
118
|
-
mailsort sync --host imap.example.com --username user@example.com --password "..."
|
|
119
|
-
mailsort train
|
|
120
|
-
mailsort sort some_label --host imap.example.com --username user@example.com --password "..."
|
|
121
|
-
mailsort predict some_label --host imap.example.com --username user@example.com --password "..."
|
|
122
|
-
```
|
|
123
|
-
|
|
124
119
|
## Python interface
|
|
125
120
|
The recommended way to use `mailsort` from Python is `MailSorter`, a small facade that wraps a
|
|
126
121
|
mailbox backend (such as `Imap`) and exposes the fetch-store-train-predict-move loop through the
|
|
@@ -4,14 +4,21 @@
|
|
|
4
4
|
[](https://mailsort.readthedocs.io/en/latest/?badge=latest)
|
|
5
5
|
[](https://github.com/psf/black)
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
|
|
7
|
+
`mailsort` reduces the interruption caused by the increase of daily emails by automatically sorting your emails into
|
|
8
|
+
folders (IMAP labels) based on the similarity of new messages to the ones you have already sorted. It uses machine
|
|
9
|
+
learning to learn from your existing folder assignments and suggests or applies folders.
|
|
9
10
|
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
11
|
+
This folder based approach allows you to focus on the most important emails first, while still being able to access less
|
|
12
|
+
important emails later. You are in contorl. When you change the folder assignments, you can retrain the `mailsort`
|
|
13
|
+
machine learning model so `mailsort` can adapt to your changing preferences. Think of it as a personal assistant that
|
|
14
|
+
helps you manage your emails.
|
|
15
|
+
|
|
16
|
+
`mailsort` is deliberately a plain Python library and command line tool rather than a hosted service: you run it
|
|
17
|
+
yourself, on your own schedule (for example from cron), against your own local database. Your emails belong to you.
|
|
18
|
+
|
|
19
|
+
`mailsort` connects to any IMAP mail server, stores your emails locally in an SQLite database, and trains a machine
|
|
20
|
+
learning model on the folders (IMAP labels) you have already assigned to your emails, and uses that model to suggest or
|
|
21
|
+
apply folders to new messages.
|
|
15
22
|
|
|
16
23
|
To learn more about `mailsort` please have a look at the documentation below.
|
|
17
24
|
|
|
@@ -81,22 +88,6 @@ password manager.
|
|
|
81
88
|
|
|
82
89
|
Run `mailsort --help` or `mailsort <command> --help` for the full list of options and examples.
|
|
83
90
|
|
|
84
|
-
### Upgrading from the pre-1.0 CLI
|
|
85
|
-
The previous flat `-u/--update` and `-l/--label` options still work exactly as before, but are
|
|
86
|
-
deprecated in favor of the subcommands above:
|
|
87
|
-
```
|
|
88
|
-
mailsort --host imap.example.com --username user@example.com --password "..." -u
|
|
89
|
-
mailsort --host imap.example.com --username user@example.com --password "..." -l "some_label"
|
|
90
|
-
mailsort --host imap.example.com --username user@example.com --password "..." -l "some_label" --dry-run
|
|
91
|
-
```
|
|
92
|
-
is equivalent to:
|
|
93
|
-
```
|
|
94
|
-
mailsort sync --host imap.example.com --username user@example.com --password "..."
|
|
95
|
-
mailsort train
|
|
96
|
-
mailsort sort some_label --host imap.example.com --username user@example.com --password "..."
|
|
97
|
-
mailsort predict some_label --host imap.example.com --username user@example.com --password "..."
|
|
98
|
-
```
|
|
99
|
-
|
|
100
91
|
## Python interface
|
|
101
92
|
The recommended way to use `mailsort` from Python is `MailSorter`, a small facade that wraps a
|
|
102
93
|
mailbox backend (such as `Imap`) and exposes the fetch-store-train-predict-move loop through the
|
|
@@ -27,7 +27,7 @@ dependencies = [
|
|
|
27
27
|
"tqdm>=4.64.1,<=4.70.1",
|
|
28
28
|
"pandas>=1.5.3,<=3.0.5",
|
|
29
29
|
"scikit-learn>=1.2.2,<=1.9.0",
|
|
30
|
-
"sqlalchemy>=2.0.0,<=2.0.
|
|
30
|
+
"sqlalchemy>=2.0.0,<=2.0.54",
|
|
31
31
|
]
|
|
32
32
|
|
|
33
33
|
[project.urls]
|
|
@@ -36,6 +36,13 @@ Homepage = "https://github.com/jan-janssen/mailsort"
|
|
|
36
36
|
[project.scripts]
|
|
37
37
|
mailsort = "mailsort.__main__:command_line_parser"
|
|
38
38
|
|
|
39
|
+
[project.optional-dependencies]
|
|
40
|
+
docs = [
|
|
41
|
+
"nbsphinx",
|
|
42
|
+
"sphinx",
|
|
43
|
+
"myst-parser",
|
|
44
|
+
]
|
|
45
|
+
|
|
39
46
|
[tool.ruff.lint]
|
|
40
47
|
select = [
|
|
41
48
|
# pycodestyle
|
|
@@ -263,31 +263,12 @@ Run `mailsort <command> --help` for the arguments of an individual command.
|
|
|
263
263
|
""",
|
|
264
264
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
265
265
|
)
|
|
266
|
-
# Top-level flags
|
|
267
|
-
#
|
|
266
|
+
# Top-level flags so they can be given before the subcommand, e.g. "mailsort --host x
|
|
267
|
+
# sync" - see _add_connection_arguments for why the subcommand copies suppress their
|
|
268
|
+
# defaults.
|
|
268
269
|
_add_connection_arguments(parser)
|
|
269
270
|
_add_database_argument(parser)
|
|
270
271
|
_add_identification_argument(parser)
|
|
271
|
-
parser.add_argument(
|
|
272
|
-
"-u",
|
|
273
|
-
"--update",
|
|
274
|
-
action="store_true",
|
|
275
|
-
help="Deprecated - use 'mailsort sync' followed by 'mailsort train' instead. "
|
|
276
|
-
"Update local database and retrain machine learning model.",
|
|
277
|
-
)
|
|
278
|
-
parser.add_argument(
|
|
279
|
-
"-l",
|
|
280
|
-
"--label",
|
|
281
|
-
help="Deprecated - use 'mailsort predict FOLDER' or 'mailsort sort FOLDER' instead. "
|
|
282
|
-
"Email label (IMAP folder) to be filtered with machine learning.",
|
|
283
|
-
)
|
|
284
|
-
parser.add_argument(
|
|
285
|
-
"-n",
|
|
286
|
-
"--dry-run",
|
|
287
|
-
action="store_true",
|
|
288
|
-
help="Deprecated - use 'mailsort predict FOLDER' instead. With -l/--label, print "
|
|
289
|
-
"the machine learning recommendations for that folder instead of moving any messages.",
|
|
290
|
-
)
|
|
291
272
|
|
|
292
273
|
subparsers = parser.add_subparsers(
|
|
293
274
|
dest="command", metavar="{sync,train,predict,sort,status,evaluate}"
|
|
@@ -557,73 +538,6 @@ def _run_evaluate(args, database, db_user_id):
|
|
|
557
538
|
return _EXIT_OK
|
|
558
539
|
|
|
559
540
|
|
|
560
|
-
def _run_legacy(args, parser, database, db_user_id):
|
|
561
|
-
"""
|
|
562
|
-
Pre-subcommand CLI, kept for backwards compatibility with existing scripts using
|
|
563
|
-
-u/--update or -l/--label - see sync/train/predict/sort for the replacements.
|
|
564
|
-
"""
|
|
565
|
-
if not args.host or not args.username:
|
|
566
|
-
print("Please provide --host and --username.")
|
|
567
|
-
return _EXIT_CONFIG_ERROR
|
|
568
|
-
if not args.password:
|
|
569
|
-
print("Please provide --password.")
|
|
570
|
-
return _EXIT_CONFIG_ERROR
|
|
571
|
-
if args.update:
|
|
572
|
-
print(
|
|
573
|
-
"mailsort: -u/--update is deprecated, use 'mailsort sync' followed by "
|
|
574
|
-
"'mailsort train' instead.",
|
|
575
|
-
file=sys.stderr,
|
|
576
|
-
)
|
|
577
|
-
imap = _connect(args, database, db_user_id)
|
|
578
|
-
try:
|
|
579
|
-
imap.update_database(quick=False)
|
|
580
|
-
imap.fit_machine_learning_model_to_database(
|
|
581
|
-
n_estimators=100,
|
|
582
|
-
max_features=400,
|
|
583
|
-
random_state=42,
|
|
584
|
-
bootstrap=True,
|
|
585
|
-
include_deleted=False,
|
|
586
|
-
)
|
|
587
|
-
finally:
|
|
588
|
-
imap.close()
|
|
589
|
-
return _EXIT_OK
|
|
590
|
-
elif args.label and args.dry_run:
|
|
591
|
-
print(
|
|
592
|
-
"mailsort: -l/--label with -n/--dry-run is deprecated, use "
|
|
593
|
-
"'mailsort predict FOLDER' instead.",
|
|
594
|
-
file=sys.stderr,
|
|
595
|
-
)
|
|
596
|
-
imap = _connect(args, database, db_user_id)
|
|
597
|
-
try:
|
|
598
|
-
recommendations = imap.get_label_recommendations(
|
|
599
|
-
label=args.label,
|
|
600
|
-
recommendation_ratio=_DEFAULT_RECOMMENDATION_RATIO,
|
|
601
|
-
label_prefix=_DEFAULT_LABEL_PREFIX,
|
|
602
|
-
)
|
|
603
|
-
finally:
|
|
604
|
-
imap.close()
|
|
605
|
-
print(_format_recommendations_table(recommendations))
|
|
606
|
-
return _EXIT_OK
|
|
607
|
-
elif args.label:
|
|
608
|
-
print(
|
|
609
|
-
"mailsort: -l/--label is deprecated, use 'mailsort sort FOLDER' instead.",
|
|
610
|
-
file=sys.stderr,
|
|
611
|
-
)
|
|
612
|
-
imap = _connect(args, database, db_user_id)
|
|
613
|
-
try:
|
|
614
|
-
imap.filter_messages_from_server(
|
|
615
|
-
label=args.label,
|
|
616
|
-
recommendation_ratio=_DEFAULT_RECOMMENDATION_RATIO,
|
|
617
|
-
label_prefix=_DEFAULT_LABEL_PREFIX,
|
|
618
|
-
)
|
|
619
|
-
finally:
|
|
620
|
-
imap.close()
|
|
621
|
-
return _EXIT_OK
|
|
622
|
-
else:
|
|
623
|
-
parser.print_help()
|
|
624
|
-
return _EXIT_OK
|
|
625
|
-
|
|
626
|
-
|
|
627
541
|
_COMMAND_HANDLERS = {
|
|
628
542
|
"sync": _run_sync,
|
|
629
543
|
"train": _run_train,
|
|
@@ -657,12 +571,12 @@ def command_line_parser(argv=None):
|
|
|
657
571
|
database = getattr(args, "database", None) or _DEFAULT_DATABASE
|
|
658
572
|
handler = _COMMAND_HANDLERS.get(args.command)
|
|
659
573
|
|
|
574
|
+
if handler is None:
|
|
575
|
+
parser.print_help()
|
|
576
|
+
return _EXIT_OK
|
|
577
|
+
|
|
660
578
|
try:
|
|
661
|
-
|
|
662
|
-
return handler(args=args, database=database, db_user_id=db_user_id)
|
|
663
|
-
return _run_legacy(
|
|
664
|
-
args=args, parser=parser, database=database, db_user_id=db_user_id
|
|
665
|
-
)
|
|
579
|
+
return handler(args=args, database=database, db_user_id=db_user_id)
|
|
666
580
|
except Exception as error:
|
|
667
581
|
print(f"mailsort: error: {error}", file=sys.stderr)
|
|
668
582
|
return _EXIT_RUNTIME_ERROR
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '0.0.
|
|
22
|
-
__version_tuple__ = version_tuple = (0, 0,
|
|
21
|
+
__version__ = version = '0.0.4'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 0, 4)
|
|
23
23
|
|
|
24
24
|
__commit_id__ = commit_id = None
|
|
@@ -1,19 +1,72 @@
|
|
|
1
1
|
from typing import Any
|
|
2
2
|
|
|
3
3
|
import pandas
|
|
4
|
-
from sqlalchemy import
|
|
4
|
+
from sqlalchemy import (
|
|
5
|
+
URL,
|
|
6
|
+
Boolean,
|
|
7
|
+
Column,
|
|
8
|
+
DateTime,
|
|
9
|
+
Engine,
|
|
10
|
+
ForeignKey,
|
|
11
|
+
Integer,
|
|
12
|
+
String,
|
|
13
|
+
Text,
|
|
14
|
+
create_engine,
|
|
15
|
+
make_url,
|
|
16
|
+
)
|
|
17
|
+
from sqlalchemy.dialects.mysql import LONGTEXT
|
|
5
18
|
from sqlalchemy.orm import Session, declarative_base
|
|
6
19
|
from tqdm import tqdm
|
|
7
20
|
|
|
8
21
|
Base = declarative_base()
|
|
9
22
|
|
|
23
|
+
# MySQL requires a length for VARCHAR columns. Identifiers, labels and single
|
|
24
|
+
# email addresses fit in VARCHAR(255), while subjects and recipient lists can
|
|
25
|
+
# exceed 255 characters and are stored as TEXT. The email content can be larger
|
|
26
|
+
# than the 64 KB limit of TEXT, so it is stored as LONGTEXT on MySQL. Since
|
|
27
|
+
# MySQL 8.4 a foreign key has to reference a unique key, so email_id is unique.
|
|
28
|
+
SHORT_STRING_LENGTH = 255
|
|
29
|
+
LongText = Text().with_variant(LONGTEXT(), "mysql")
|
|
30
|
+
|
|
31
|
+
# MySQL's utf8 charset only supports characters of up to 3 bytes, so emojis in
|
|
32
|
+
# subjects or content require utf8mb4. The binary collation keeps comparisons
|
|
33
|
+
# case-sensitive, like in SQLite.
|
|
34
|
+
MYSQL_TABLE_ARGS = {"mysql_charset": "utf8mb4", "mysql_collate": "utf8mb4_bin"}
|
|
35
|
+
_MYSQL_CHARSET_DRIVERS = ("mysqldb", "pymysql")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _with_mysql_charset(connection_str: str | URL) -> URL:
|
|
39
|
+
"""
|
|
40
|
+
Add charset=utf8mb4 to MySQL connection strings without an explicit charset,
|
|
41
|
+
so 4 byte characters are also transmitted correctly.
|
|
42
|
+
"""
|
|
43
|
+
url = make_url(connection_str)
|
|
44
|
+
if url.get_driver_name() in _MYSQL_CHARSET_DRIVERS and "charset" not in url.query:
|
|
45
|
+
url = url.update_query_dict({"charset": "utf8mb4"})
|
|
46
|
+
return url
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def create_database_engine(connection_str: str | URL) -> Engine:
|
|
50
|
+
return create_engine(_with_mysql_charset(connection_str))
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _missing_to_none(value: Any) -> Any:
|
|
54
|
+
"""
|
|
55
|
+
Convert missing values in a pandas.DataFrame, like NaN or NaT, to None, as
|
|
56
|
+
MySQL rejects NaN while None is stored as NULL.
|
|
57
|
+
"""
|
|
58
|
+
if pandas.api.types.is_scalar(value) and pandas.isna(value):
|
|
59
|
+
return None
|
|
60
|
+
return value
|
|
61
|
+
|
|
10
62
|
|
|
11
63
|
class EmailContent(Base):
|
|
12
64
|
__tablename__ = "email_content"
|
|
65
|
+
__table_args__ = MYSQL_TABLE_ARGS
|
|
13
66
|
id = Column(Integer, primary_key=True)
|
|
14
|
-
email_id = Column(String)
|
|
15
|
-
email_subject = Column(
|
|
16
|
-
email_content = Column(
|
|
67
|
+
email_id = Column(String(SHORT_STRING_LENGTH), unique=True)
|
|
68
|
+
email_subject = Column(Text)
|
|
69
|
+
email_content = Column(LongText)
|
|
17
70
|
email_deleted = Column(Boolean)
|
|
18
71
|
email_date = Column(DateTime)
|
|
19
72
|
user_id = Column(Integer)
|
|
@@ -21,41 +74,46 @@ class EmailContent(Base):
|
|
|
21
74
|
|
|
22
75
|
class Threads(Base):
|
|
23
76
|
__tablename__ = "email_threads"
|
|
77
|
+
__table_args__ = MYSQL_TABLE_ARGS
|
|
24
78
|
id = Column(Integer, primary_key=True)
|
|
25
|
-
email_id = Column(String, ForeignKey("email_content.email_id"))
|
|
26
|
-
thread_id = Column(String)
|
|
79
|
+
email_id = Column(String(SHORT_STRING_LENGTH), ForeignKey("email_content.email_id"))
|
|
80
|
+
thread_id = Column(String(SHORT_STRING_LENGTH))
|
|
27
81
|
user_id = Column(Integer)
|
|
28
82
|
|
|
29
83
|
|
|
30
84
|
class Labels(Base):
|
|
31
85
|
__tablename__ = "email_labels"
|
|
86
|
+
__table_args__ = MYSQL_TABLE_ARGS
|
|
32
87
|
id = Column(Integer, primary_key=True)
|
|
33
|
-
email_id = Column(String, ForeignKey("email_content.email_id"))
|
|
34
|
-
label_id = Column(String)
|
|
88
|
+
email_id = Column(String(SHORT_STRING_LENGTH), ForeignKey("email_content.email_id"))
|
|
89
|
+
label_id = Column(String(SHORT_STRING_LENGTH))
|
|
35
90
|
user_id = Column(Integer)
|
|
36
91
|
|
|
37
92
|
|
|
38
93
|
class EmailTo(Base):
|
|
39
94
|
__tablename__ = "email_to"
|
|
95
|
+
__table_args__ = MYSQL_TABLE_ARGS
|
|
40
96
|
id = Column(Integer, primary_key=True)
|
|
41
|
-
email_id = Column(String, ForeignKey("email_content.email_id"))
|
|
42
|
-
email_to = Column(
|
|
97
|
+
email_id = Column(String(SHORT_STRING_LENGTH), ForeignKey("email_content.email_id"))
|
|
98
|
+
email_to = Column(Text)
|
|
43
99
|
user_id = Column(Integer)
|
|
44
100
|
|
|
45
101
|
|
|
46
102
|
class EmailCc(Base):
|
|
47
103
|
__tablename__ = "email_cc"
|
|
104
|
+
__table_args__ = MYSQL_TABLE_ARGS
|
|
48
105
|
id = Column(Integer, primary_key=True)
|
|
49
|
-
email_id = Column(String, ForeignKey("email_content.email_id"))
|
|
50
|
-
email_cc = Column(
|
|
106
|
+
email_id = Column(String(SHORT_STRING_LENGTH), ForeignKey("email_content.email_id"))
|
|
107
|
+
email_cc = Column(Text)
|
|
51
108
|
user_id = Column(Integer)
|
|
52
109
|
|
|
53
110
|
|
|
54
111
|
class EmailFrom(Base):
|
|
55
112
|
__tablename__ = "email_from"
|
|
113
|
+
__table_args__ = MYSQL_TABLE_ARGS
|
|
56
114
|
id = Column(Integer, primary_key=True)
|
|
57
|
-
email_id = Column(String, ForeignKey("email_content.email_id"))
|
|
58
|
-
email_from = Column(String)
|
|
115
|
+
email_id = Column(String(SHORT_STRING_LENGTH), ForeignKey("email_content.email_id"))
|
|
116
|
+
email_from = Column(String(SHORT_STRING_LENGTH))
|
|
59
117
|
user_id = Column(Integer)
|
|
60
118
|
|
|
61
119
|
|
|
@@ -333,7 +391,11 @@ class DatabaseInterface(DatabaseTemplate):
|
|
|
333
391
|
def _commit_thread_table(self, df: pandas.DataFrame, user_id: int = 1) -> None:
|
|
334
392
|
self._session.add_all(
|
|
335
393
|
[
|
|
336
|
-
Threads(
|
|
394
|
+
Threads(
|
|
395
|
+
email_id=email_id,
|
|
396
|
+
thread_id=_missing_to_none(thread_id),
|
|
397
|
+
user_id=user_id,
|
|
398
|
+
)
|
|
337
399
|
for email_id, thread_id in zip(df["id"], df["threads"], strict=False)
|
|
338
400
|
]
|
|
339
401
|
)
|
|
@@ -342,7 +404,11 @@ class DatabaseInterface(DatabaseTemplate):
|
|
|
342
404
|
def _commit_email_from_table(self, df: pandas.DataFrame, user_id: int = 1) -> None:
|
|
343
405
|
self._session.add_all(
|
|
344
406
|
[
|
|
345
|
-
EmailFrom(
|
|
407
|
+
EmailFrom(
|
|
408
|
+
email_id=email_id,
|
|
409
|
+
email_from=_missing_to_none(email_from),
|
|
410
|
+
user_id=user_id,
|
|
411
|
+
)
|
|
346
412
|
for email_id, email_from in zip(df["id"], df["from"], strict=False)
|
|
347
413
|
]
|
|
348
414
|
)
|
|
@@ -383,10 +449,10 @@ class DatabaseInterface(DatabaseTemplate):
|
|
|
383
449
|
[
|
|
384
450
|
EmailContent(
|
|
385
451
|
email_id=email_id,
|
|
386
|
-
email_subject=email_subject,
|
|
387
|
-
email_content=email_content,
|
|
452
|
+
email_subject=_missing_to_none(email_subject),
|
|
453
|
+
email_content=_missing_to_none(email_content),
|
|
388
454
|
email_deleted=False,
|
|
389
|
-
email_date=email_date,
|
|
455
|
+
email_date=_missing_to_none(email_date),
|
|
390
456
|
user_id=user_id,
|
|
391
457
|
)
|
|
392
458
|
for email_id, email_subject, email_content, email_date in zip(
|
|
@@ -6,6 +6,7 @@ from typing import Any
|
|
|
6
6
|
|
|
7
7
|
_MAX_DATE_COMMAS = 2
|
|
8
8
|
_DATE_HYPHEN_COUNT = 2
|
|
9
|
+
_WEEKDAY_ABBREVIATION_LENGTH = 3
|
|
9
10
|
|
|
10
11
|
|
|
11
12
|
# https://stackoverflow.com/questions/753052/strip-html-from-strings-in-python
|
|
@@ -37,6 +38,9 @@ def email_date_converter(email_date: Any) -> datetime | None:
|
|
|
37
38
|
email_date = email_date.replace("\xa0", "")
|
|
38
39
|
if email_date.count(",") >= _MAX_DATE_COMMAS:
|
|
39
40
|
email_date = ", ".join(email_date.split(", ")[-2:])
|
|
41
|
+
weekday, separator, remainder = email_date.partition(", ")
|
|
42
|
+
if separator and weekday.isalpha() and len(weekday) != _WEEKDAY_ABBREVIATION_LENGTH:
|
|
43
|
+
email_date = remainder
|
|
40
44
|
if email_date[-3:-2].isalpha():
|
|
41
45
|
email_date = " ".join(email_date.split()[:-1])
|
|
42
46
|
if email_date[-1].isalpha():
|
|
@@ -8,10 +8,9 @@ a mail backend such as mailsort.Imap, which would require live mail server crede
|
|
|
8
8
|
not need.
|
|
9
9
|
"""
|
|
10
10
|
|
|
11
|
-
from sqlalchemy import create_engine
|
|
12
11
|
from sqlalchemy.orm import sessionmaker
|
|
13
12
|
|
|
14
|
-
from mailsort.base import get_email_database
|
|
13
|
+
from mailsort.base import create_database_engine, get_email_database
|
|
15
14
|
from mailsort.ml.evaluation import (
|
|
16
15
|
EvaluationReport,
|
|
17
16
|
FolderMetrics,
|
|
@@ -76,7 +75,7 @@ def evaluate_models(
|
|
|
76
75
|
Returns:
|
|
77
76
|
mailsort.ml.evaluation.EvaluationReport
|
|
78
77
|
"""
|
|
79
|
-
engine =
|
|
78
|
+
engine = create_database_engine(connection_str)
|
|
80
79
|
session = sessionmaker(bind=engine)()
|
|
81
80
|
try:
|
|
82
81
|
db_email = get_email_database(engine=engine, session=session)
|
|
@@ -131,7 +130,7 @@ def score_holdout(
|
|
|
131
130
|
Returns:
|
|
132
131
|
mailsort.ml.evaluation.HoldoutScores
|
|
133
132
|
"""
|
|
134
|
-
engine =
|
|
133
|
+
engine = create_database_engine(connection_str)
|
|
135
134
|
session = sessionmaker(bind=engine)()
|
|
136
135
|
try:
|
|
137
136
|
db_email = get_email_database(engine=engine, session=session)
|
|
@@ -3,10 +3,9 @@ import email
|
|
|
3
3
|
import imaplib
|
|
4
4
|
import re
|
|
5
5
|
|
|
6
|
-
from sqlalchemy import create_engine
|
|
7
6
|
from sqlalchemy.orm import sessionmaker
|
|
8
7
|
|
|
9
|
-
from mailsort.base import get_email_database
|
|
8
|
+
from mailsort.base import create_database_engine, get_email_database
|
|
10
9
|
from mailsort.base.mail import AbstractMailBox
|
|
11
10
|
from mailsort.imap.message import get_email_dict
|
|
12
11
|
from mailsort.ml import get_machine_learning_database
|
|
@@ -341,7 +340,7 @@ class ImapMailBase(AbstractMailBox):
|
|
|
341
340
|
|
|
342
341
|
@staticmethod
|
|
343
342
|
def _create_databases(connection_str):
|
|
344
|
-
engine =
|
|
343
|
+
engine = create_database_engine(connection_str)
|
|
345
344
|
session = sessionmaker(bind=engine)()
|
|
346
345
|
db_email = get_email_database(engine=engine, session=session)
|
|
347
346
|
db_ml = get_machine_learning_database(engine=engine, session=session)
|
|
@@ -1,26 +1,53 @@
|
|
|
1
1
|
import pickle
|
|
2
|
+
import zlib
|
|
2
3
|
|
|
3
4
|
from sklearn.ensemble import RandomForestClassifier
|
|
4
|
-
from sqlalchemy import Column, Engine, Integer, String
|
|
5
|
+
from sqlalchemy import Column, Engine, Integer, LargeBinary, String, Text
|
|
6
|
+
from sqlalchemy.dialects.mysql import LONGBLOB
|
|
5
7
|
from sqlalchemy.orm import Session, declarative_base
|
|
6
8
|
|
|
7
|
-
from mailsort.base.database import
|
|
9
|
+
from mailsort.base.database import (
|
|
10
|
+
MYSQL_TABLE_ARGS,
|
|
11
|
+
SHORT_STRING_LENGTH,
|
|
12
|
+
DatabaseTemplate,
|
|
13
|
+
)
|
|
8
14
|
|
|
9
15
|
Base = declarative_base()
|
|
10
16
|
|
|
17
|
+
# Pickled random forest models easily exceed the 16 MB limit of MEDIUMBLOB
|
|
18
|
+
LongBinary = LargeBinary().with_variant(LONGBLOB(), "mysql")
|
|
19
|
+
|
|
20
|
+
# MySQL rejects statements larger than max_allowed_packet (64 MB by default and
|
|
21
|
+
# at most 1 GB), while a pickled calibrated random forest can be several hundred
|
|
22
|
+
# MB. So the compressed pickle is split into chunks, stored as individual rows.
|
|
23
|
+
MODEL_CHUNK_SIZE = 1024 * 1024
|
|
24
|
+
|
|
11
25
|
|
|
12
26
|
class MachineLearningLabels(Base):
|
|
13
27
|
__tablename__ = "ml_labels"
|
|
28
|
+
__table_args__ = MYSQL_TABLE_ARGS
|
|
29
|
+
id = Column(Integer, primary_key=True)
|
|
30
|
+
label_id = Column(String(SHORT_STRING_LENGTH))
|
|
31
|
+
random_forest = Column(LongBinary)
|
|
32
|
+
user_id = Column(Integer)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class MachineLearningModelChunks(Base):
|
|
36
|
+
__tablename__ = "ml_label_chunks"
|
|
37
|
+
__table_args__ = MYSQL_TABLE_ARGS
|
|
14
38
|
id = Column(Integer, primary_key=True)
|
|
15
|
-
label_id = Column(String)
|
|
16
|
-
|
|
39
|
+
label_id = Column(String(SHORT_STRING_LENGTH))
|
|
40
|
+
chunk_index = Column(Integer)
|
|
41
|
+
data = Column(LongBinary)
|
|
17
42
|
user_id = Column(Integer)
|
|
18
43
|
|
|
19
44
|
|
|
20
45
|
class MachineLearningFeatures(Base):
|
|
21
46
|
__tablename__ = "ml_features"
|
|
47
|
+
__table_args__ = MYSQL_TABLE_ARGS
|
|
22
48
|
id = Column(Integer, primary_key=True)
|
|
23
|
-
|
|
49
|
+
# Features are derived from the recipients, which can exceed 255 characters
|
|
50
|
+
feature = Column(Text)
|
|
24
51
|
user_id = Column(Integer)
|
|
25
52
|
|
|
26
53
|
|
|
@@ -68,12 +95,12 @@ class MachineLearningDatabase(DatabaseTemplate):
|
|
|
68
95
|
if len(model_dict_new) > 0:
|
|
69
96
|
self._session.add_all(
|
|
70
97
|
[
|
|
71
|
-
MachineLearningLabels(
|
|
72
|
-
|
|
73
|
-
)
|
|
74
|
-
for k, v in model_dict_new.items()
|
|
98
|
+
MachineLearningLabels(label_id=k, user_id=user_id)
|
|
99
|
+
for k in model_dict_new
|
|
75
100
|
]
|
|
76
101
|
)
|
|
102
|
+
for k, v in model_dict_new.items():
|
|
103
|
+
self._store_model_chunks(label=k, model=v, user_id=user_id)
|
|
77
104
|
if len(feature_new_lst) > 0:
|
|
78
105
|
self._session.add_all(
|
|
79
106
|
[
|
|
@@ -91,13 +118,17 @@ class MachineLearningDatabase(DatabaseTemplate):
|
|
|
91
118
|
.all()
|
|
92
119
|
)
|
|
93
120
|
for label_obj in label_obj_lst:
|
|
94
|
-
label_obj.random_forest =
|
|
95
|
-
|
|
96
|
-
)
|
|
121
|
+
label_obj.random_forest = None
|
|
122
|
+
self._delete_model_chunks(
|
|
123
|
+
label_lst=list(model_dict_update.keys()), user_id=user_id
|
|
124
|
+
)
|
|
125
|
+
for k, v in model_dict_update.items():
|
|
126
|
+
self._store_model_chunks(label=k, model=v, user_id=user_id)
|
|
97
127
|
if len(model_delete_lst) > 0:
|
|
98
128
|
self._session.query(MachineLearningLabels).filter(
|
|
99
129
|
MachineLearningLabels.user_id == user_id
|
|
100
130
|
).filter(MachineLearningLabels.label_id.in_(model_delete_lst)).delete()
|
|
131
|
+
self._delete_model_chunks(label_lst=model_delete_lst, user_id=user_id)
|
|
101
132
|
if len(feature_remove_lst) > 0:
|
|
102
133
|
self._session.query(MachineLearningFeatures).filter(
|
|
103
134
|
MachineLearningFeatures.user_id == user_id
|
|
@@ -124,10 +155,45 @@ class MachineLearningDatabase(DatabaseTemplate):
|
|
|
124
155
|
)
|
|
125
156
|
feature_lst = self.get_features(user_id=user_id)
|
|
126
157
|
return {
|
|
127
|
-
label_obj.label_id:
|
|
158
|
+
label_obj.label_id: (
|
|
159
|
+
# Models stored before the chunked storage was introduced
|
|
160
|
+
pickle.loads(label_obj.random_forest)
|
|
161
|
+
if label_obj.random_forest is not None
|
|
162
|
+
else self._load_model_chunks(label=label_obj.label_id, user_id=user_id)
|
|
163
|
+
)
|
|
128
164
|
for label_obj in label_obj_lst
|
|
129
165
|
}, feature_lst
|
|
130
166
|
|
|
167
|
+
def _store_model_chunks(self, label: str, model, user_id: int = 1) -> None:
|
|
168
|
+
data = zlib.compress(pickle.dumps(model))
|
|
169
|
+
for chunk_index, start in enumerate(range(0, len(data), MODEL_CHUNK_SIZE)):
|
|
170
|
+
self._session.add(
|
|
171
|
+
MachineLearningModelChunks(
|
|
172
|
+
label_id=label,
|
|
173
|
+
chunk_index=chunk_index,
|
|
174
|
+
data=data[start : start + MODEL_CHUNK_SIZE],
|
|
175
|
+
user_id=user_id,
|
|
176
|
+
)
|
|
177
|
+
)
|
|
178
|
+
# Flush each chunk individually, so the chunks are not combined in
|
|
179
|
+
# a single multi-row INSERT statement exceeding max_allowed_packet
|
|
180
|
+
self._session.flush()
|
|
181
|
+
|
|
182
|
+
def _load_model_chunks(self, label: str, user_id: int = 1):
|
|
183
|
+
chunk_lst = (
|
|
184
|
+
self._session.query(MachineLearningModelChunks.data)
|
|
185
|
+
.filter(MachineLearningModelChunks.user_id == user_id)
|
|
186
|
+
.filter(MachineLearningModelChunks.label_id == label)
|
|
187
|
+
.order_by(MachineLearningModelChunks.chunk_index)
|
|
188
|
+
.all()
|
|
189
|
+
)
|
|
190
|
+
return pickle.loads(zlib.decompress(b"".join(chunk[0] for chunk in chunk_lst)))
|
|
191
|
+
|
|
192
|
+
def _delete_model_chunks(self, label_lst: list[str], user_id: int = 1) -> None:
|
|
193
|
+
self._session.query(MachineLearningModelChunks).filter(
|
|
194
|
+
MachineLearningModelChunks.user_id == user_id
|
|
195
|
+
).filter(MachineLearningModelChunks.label_id.in_(label_lst)).delete()
|
|
196
|
+
|
|
131
197
|
def get_labels(self, user_id: int = 1) -> list[str]:
|
|
132
198
|
"""
|
|
133
199
|
List the folders/labels a machine learning model has been trained for.
|
|
@@ -8,10 +8,9 @@ connection to the mail server.
|
|
|
8
8
|
|
|
9
9
|
from dataclasses import dataclass
|
|
10
10
|
|
|
11
|
-
from sqlalchemy import create_engine
|
|
12
11
|
from sqlalchemy.orm import sessionmaker
|
|
13
12
|
|
|
14
|
-
from mailsort.base import get_email_database
|
|
13
|
+
from mailsort.base import create_database_engine, get_email_database
|
|
15
14
|
from mailsort.ml import get_machine_learning_database
|
|
16
15
|
|
|
17
16
|
|
|
@@ -39,7 +38,7 @@ def get_database_status(connection_str, db_user_id=1):
|
|
|
39
38
|
Returns:
|
|
40
39
|
DatabaseStatus
|
|
41
40
|
"""
|
|
42
|
-
engine =
|
|
41
|
+
engine = create_database_engine(connection_str)
|
|
43
42
|
session = sessionmaker(bind=engine)()
|
|
44
43
|
try:
|
|
45
44
|
db_email = get_email_database(engine=engine, session=session)
|
|
@@ -8,10 +8,9 @@ instead of going through a mail backend such as mailsort.Imap, which would requi
|
|
|
8
8
|
server credentials it does not need.
|
|
9
9
|
"""
|
|
10
10
|
|
|
11
|
-
from sqlalchemy import create_engine
|
|
12
11
|
from sqlalchemy.orm import sessionmaker
|
|
13
12
|
|
|
14
|
-
from mailsort.base import get_email_database
|
|
13
|
+
from mailsort.base import create_database_engine, get_email_database
|
|
15
14
|
from mailsort.ml import (
|
|
16
15
|
DEFAULT_MAX_CALIBRATION_CV_FOLDS,
|
|
17
16
|
DEFAULT_MIN_SAMPLES_PER_CLASS_FOR_CALIBRATION,
|
|
@@ -58,7 +57,7 @@ def train_machine_learning_models(
|
|
|
58
57
|
Returns:
|
|
59
58
|
int: number of per-folder models trained
|
|
60
59
|
"""
|
|
61
|
-
engine =
|
|
60
|
+
engine = create_database_engine(connection_str)
|
|
62
61
|
session = sessionmaker(bind=engine)()
|
|
63
62
|
try:
|
|
64
63
|
db_email = get_email_database(engine=engine, session=session)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|