fivefifty 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fivefifty-0.1.0/LICENSE +21 -0
- fivefifty-0.1.0/PKG-INFO +114 -0
- fivefifty-0.1.0/README.md +103 -0
- fivefifty-0.1.0/fivefifty/__init__.py +0 -0
- fivefifty-0.1.0/fivefifty/classify.py +95 -0
- fivefifty-0.1.0/fivefifty/cli.py +58 -0
- fivefifty-0.1.0/fivefifty/evaluate.py +58 -0
- fivefifty-0.1.0/fivefifty/experiments.py +170 -0
- fivefifty-0.1.0/fivefifty/loader.py +28 -0
- fivefifty-0.1.0/fivefifty/model.json +1 -0
- fivefifty-0.1.0/fivefifty/model.py +28 -0
- fivefifty-0.1.0/fivefifty/parse.py +44 -0
- fivefifty-0.1.0/fivefifty/split.py +25 -0
- fivefifty-0.1.0/fivefifty/tokenizer.py +60 -0
- fivefifty-0.1.0/fivefifty/train.py +63 -0
- fivefifty-0.1.0/fivefifty.egg-info/PKG-INFO +114 -0
- fivefifty-0.1.0/fivefifty.egg-info/SOURCES.txt +20 -0
- fivefifty-0.1.0/fivefifty.egg-info/dependency_links.txt +1 -0
- fivefifty-0.1.0/fivefifty.egg-info/entry_points.txt +2 -0
- fivefifty-0.1.0/fivefifty.egg-info/top_level.txt +1 -0
- fivefifty-0.1.0/pyproject.toml +18 -0
- fivefifty-0.1.0/setup.cfg +4 -0
fivefifty-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 AdJakub
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
fivefifty-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: fivefifty
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A spam filter you can read, that tells you why, and that learns from your own inbox
|
|
5
|
+
Author: AdJak
|
|
6
|
+
License: MIT
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Dynamic: license-file
|
|
11
|
+
|
|
12
|
+
# fivefifty
|
|
13
|
+
|
|
14
|
+
> A spam filter you can read, that tells you why, and that learns from your own inbox.
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
fivefifty is a tiny naive bayes spam filter, written from scratch. It utilizes no machine learning or non-standard libraries at all. I made it to learn about spam filters and provide a learning tool for others learning about spam filters.
|
|
18
|
+
|
|
19
|
+
## install
|
|
20
|
+
```
|
|
21
|
+
pip install fivefifty
|
|
22
|
+
```
|
|
23
|
+
or from source:
|
|
24
|
+
```
|
|
25
|
+
git clone https://github.com/AdJakub/fivefifty
|
|
26
|
+
cd fivefifty
|
|
27
|
+
pip install -e .
|
|
28
|
+
```
|
|
29
|
+
needs python 3.10+. zero dependencies, standard library only.
|
|
30
|
+
|
|
31
|
+
## use it
|
|
32
|
+
check one email (a `.eml` file, e.g. gmail → ⋮ → "Download message"):
|
|
33
|
+
```
|
|
34
|
+
fivefifty check some_email.eml
|
|
35
|
+
```
|
|
36
|
+
```
|
|
37
|
+
SPAM
|
|
38
|
+
[('payment', 12.719685971180983), ('le', 15.667823491997437), ('de', 19.956307904829593), ('les', 20.702822876133904), ('your', 26.43092106142582)]
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
train your own model on your own mail:
|
|
42
|
+
```
|
|
43
|
+
fivefifty train my_spam/ my_ham/ --out my_model.json
|
|
44
|
+
fivefifty check some_email.eml --model my_model.json
|
|
45
|
+
```
|
|
46
|
+
The built in model is based on mostly english 2002 spam. Training your own shows it modern spam in your own language
|
|
47
|
+
|
|
48
|
+
everything runs locally. no email ever leaves your machine.
|
|
49
|
+
|
|
50
|
+
### getting your mail into folders
|
|
51
|
+
the easiest way right now is thunderbird (free, works with gmail, outlook and most other providers):
|
|
52
|
+
|
|
53
|
+
1. install [thunderbird](https://www.thunderbird.net) and add your email account
|
|
54
|
+
2. open your **spam** folder (for gmail it shows up as `[Gmail]` → `Spam`), select all with `ctrl+a`, right-click → **Save As…**, and pick an empty folder like `my_spam/`. every email gets saved as its own `.eml` file.
|
|
55
|
+
3. do the same for your **inbox** into `my_ham/`
|
|
56
|
+
4. train:
|
|
57
|
+
```
|
|
58
|
+
fivefifty train my_spam/ my_ham/ --out my_model.json
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
tips:
|
|
62
|
+
- gmail deletes spam after 30 days, so you'll usually have a lot less spam than ham. that's fine, the filter accounts for it.
|
|
63
|
+
- only put emails you're sure about in each folder. a few mislabeled ones teach it the wrong words.
|
|
64
|
+
- your emails and your model stay on your computer. don't share `my_model.json`: it's built from your private mail.
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
## results
|
|
68
|
+
test set: 100 spam + 500 ham the filter never saw while training.
|
|
69
|
+
|
|
70
|
+
| version | spam caught | spam missed | real emails flagged | precision | recall |
|
|
71
|
+
|---|---|---|---|---|---|
|
|
72
|
+
| first try (bernoulli-ish, present words only) | 100 | 0 | 104 | 49.0% | 100% |
|
|
73
|
+
| bernoulli NB, all words | 21 | 79 | 5 | 80.8% | 21.0% |
|
|
74
|
+
| bernoulli NB, words seen in ≥112 emails | 84 | 16 | 14 | 85.7% | 84.0% |
|
|
75
|
+
| **multinomial NB + MIME decoding (final)** | **93** | **7** | **0** | **100%** | **93.0%** |
|
|
76
|
+
|
|
77
|
+
Accuracy isn't the main number to look at, the test set had more ham than spam emails, and saying ham always gives 83% accuracy.
|
|
78
|
+
## how it works
|
|
79
|
+
1. `parse.py`: unpacks the email (MIME: base64, quoted-printable, multipart), keeps subject + text parts. no headers, on purpose.
|
|
80
|
+
2. `tokenizer.py`: drops `<style>` blocks and html tags, lowercases, splits into words, drops "words" over 22 chars (base64 junk)
|
|
81
|
+
3. `train.py`: counts every word in spam and in ham, turns counts into smoothed fractions
|
|
82
|
+
4. `classify.py`: multinomial naive bayes, adds up log-evidence for every word in the email. `explain()` shows which words pushed hardest.
|
|
83
|
+
5. `evaluate.py`: caught / missed / false positives, precision, recall
|
|
84
|
+
6. `cli.py`: the `fivefifty` command
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
The filter started out as a bernoulli naive bayes filter. However ham started 216 points ahead because of 35k junk words and it only had good results when cutting a very large part of the vocabulary. That's when the filter was switched to a multinomial algorithm instead. A large vocabulary base does not hurt multinomial naive bayes algorithms.
|
|
88
|
+
|
|
89
|
+
## how it was measured
|
|
90
|
+
- emails split into train (2400) and test (600), spam and ham split separately, seeded
|
|
91
|
+
- train split again into fit (1920) + validation (480) for every choice i made (like N for bernoulli)
|
|
92
|
+
- test was only used to report final numbers once per model, plus once more after adding MIME decoding (a decision made on validation, not on test)
|
|
93
|
+
- the model shipped in the package is trained on all 3000 emails
|
|
94
|
+
|
|
95
|
+
## what it found along the way
|
|
96
|
+
- 64% of the vocabulary was words seen in exactly one email (mostly base64 junk)
|
|
97
|
+
- bernoulli NB works best with a tiny vocabulary (~1%), multinomial doesnt care. matches McCallum & Nigam (1998)
|
|
98
|
+
- on a real 2026 phishing email, french "poisoning" text backfired: french words look spammy to a filter trained on 2002 email
|
|
99
|
+
|
|
100
|
+
## limitations
|
|
101
|
+
- trained on the 2002 SpamAssassin corpus. modern spam is different, so train your own for real use.
|
|
102
|
+
- only english-ish easy_ham was measured. hard_ham (newsletters, html-heavy) not evaluated yet.
|
|
103
|
+
- 0 false positives in 500 means the real rate is probably under ~0.6% (rule of three), not zero
|
|
104
|
+
- some mailing lists break base64 bodies by appending footers. those cant be decoded.
|
|
105
|
+
- no learning from corrections yet
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
## data
|
|
110
|
+
built and measured on the [SpamAssassin public corpus](https://spamassassin.apache.org/old/publiccorpus/) (`20021010_spam`, `20021010_easy_ham`, `20021010_hard_ham`).
|
|
111
|
+
to reproduce: extract into `data/spam`, `data/easy_ham`, `data/hard_ham`, then `python -m fivefifty.evaluate` from the repo root.
|
|
112
|
+
|
|
113
|
+
## license
|
|
114
|
+
MIT
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
# fivefifty
|
|
2
|
+
|
|
3
|
+
> A spam filter you can read, that tells you why, and that learns from your own inbox.
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
fivefifty is a tiny naive bayes spam filter, written from scratch. It utilizes no machine learning or non-standard libraries at all. I made it to learn about spam filters and provide a learning tool for others learning about spam filters.
|
|
7
|
+
|
|
8
|
+
## install
|
|
9
|
+
```
|
|
10
|
+
pip install fivefifty
|
|
11
|
+
```
|
|
12
|
+
or from source:
|
|
13
|
+
```
|
|
14
|
+
git clone https://github.com/AdJakub/fivefifty
|
|
15
|
+
cd fivefifty
|
|
16
|
+
pip install -e .
|
|
17
|
+
```
|
|
18
|
+
needs python 3.10+. zero dependencies, standard library only.
|
|
19
|
+
|
|
20
|
+
## use it
|
|
21
|
+
check one email (a `.eml` file, e.g. gmail → ⋮ → "Download message"):
|
|
22
|
+
```
|
|
23
|
+
fivefifty check some_email.eml
|
|
24
|
+
```
|
|
25
|
+
```
|
|
26
|
+
SPAM
|
|
27
|
+
[('payment', 12.719685971180983), ('le', 15.667823491997437), ('de', 19.956307904829593), ('les', 20.702822876133904), ('your', 26.43092106142582)]
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
train your own model on your own mail:
|
|
31
|
+
```
|
|
32
|
+
fivefifty train my_spam/ my_ham/ --out my_model.json
|
|
33
|
+
fivefifty check some_email.eml --model my_model.json
|
|
34
|
+
```
|
|
35
|
+
The built in model is based on mostly english 2002 spam. Training your own shows it modern spam in your own language
|
|
36
|
+
|
|
37
|
+
everything runs locally. no email ever leaves your machine.
|
|
38
|
+
|
|
39
|
+
### getting your mail into folders
|
|
40
|
+
the easiest way right now is thunderbird (free, works with gmail, outlook and most other providers):
|
|
41
|
+
|
|
42
|
+
1. install [thunderbird](https://www.thunderbird.net) and add your email account
|
|
43
|
+
2. open your **spam** folder (for gmail it shows up as `[Gmail]` → `Spam`), select all with `ctrl+a`, right-click → **Save As…**, and pick an empty folder like `my_spam/`. every email gets saved as its own `.eml` file.
|
|
44
|
+
3. do the same for your **inbox** into `my_ham/`
|
|
45
|
+
4. train:
|
|
46
|
+
```
|
|
47
|
+
fivefifty train my_spam/ my_ham/ --out my_model.json
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
tips:
|
|
51
|
+
- gmail deletes spam after 30 days, so you'll usually have a lot less spam than ham. that's fine, the filter accounts for it.
|
|
52
|
+
- only put emails you're sure about in each folder. a few mislabeled ones teach it the wrong words.
|
|
53
|
+
- your emails and your model stay on your computer. don't share `my_model.json`: it's built from your private mail.
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
## results
|
|
57
|
+
test set: 100 spam + 500 ham the filter never saw while training.
|
|
58
|
+
|
|
59
|
+
| version | spam caught | spam missed | real emails flagged | precision | recall |
|
|
60
|
+
|---|---|---|---|---|---|
|
|
61
|
+
| first try (bernoulli-ish, present words only) | 100 | 0 | 104 | 49.0% | 100% |
|
|
62
|
+
| bernoulli NB, all words | 21 | 79 | 5 | 80.8% | 21.0% |
|
|
63
|
+
| bernoulli NB, words seen in ≥112 emails | 84 | 16 | 14 | 85.7% | 84.0% |
|
|
64
|
+
| **multinomial NB + MIME decoding (final)** | **93** | **7** | **0** | **100%** | **93.0%** |
|
|
65
|
+
|
|
66
|
+
Accuracy isn't the main number to look at, the test set had more ham than spam emails, and saying ham always gives 83% accuracy.
|
|
67
|
+
## how it works
|
|
68
|
+
1. `parse.py`: unpacks the email (MIME: base64, quoted-printable, multipart), keeps subject + text parts. no headers, on purpose.
|
|
69
|
+
2. `tokenizer.py`: drops `<style>` blocks and html tags, lowercases, splits into words, drops "words" over 22 chars (base64 junk)
|
|
70
|
+
3. `train.py`: counts every word in spam and in ham, turns counts into smoothed fractions
|
|
71
|
+
4. `classify.py`: multinomial naive bayes, adds up log-evidence for every word in the email. `explain()` shows which words pushed hardest.
|
|
72
|
+
5. `evaluate.py`: caught / missed / false positives, precision, recall
|
|
73
|
+
6. `cli.py`: the `fivefifty` command
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
The filter started out as a bernoulli naive bayes filter. However ham started 216 points ahead because of 35k junk words and it only had good results when cutting a very large part of the vocabulary. That's when the filter was switched to a multinomial algorithm instead. A large vocabulary base does not hurt multinomial naive bayes algorithms.
|
|
77
|
+
|
|
78
|
+
## how it was measured
|
|
79
|
+
- emails split into train (2400) and test (600), spam and ham split separately, seeded
|
|
80
|
+
- train split again into fit (1920) + validation (480) for every choice i made (like N for bernoulli)
|
|
81
|
+
- test was only used to report final numbers once per model, plus once more after adding MIME decoding (a decision made on validation, not on test)
|
|
82
|
+
- the model shipped in the package is trained on all 3000 emails
|
|
83
|
+
|
|
84
|
+
## what it found along the way
|
|
85
|
+
- 64% of the vocabulary was words seen in exactly one email (mostly base64 junk)
|
|
86
|
+
- bernoulli NB works best with a tiny vocabulary (~1%), multinomial doesnt care. matches McCallum & Nigam (1998)
|
|
87
|
+
- on a real 2026 phishing email, french "poisoning" text backfired: french words look spammy to a filter trained on 2002 email
|
|
88
|
+
|
|
89
|
+
## limitations
|
|
90
|
+
- trained on the 2002 SpamAssassin corpus. modern spam is different, so train your own for real use.
|
|
91
|
+
- only english-ish easy_ham was measured. hard_ham (newsletters, html-heavy) not evaluated yet.
|
|
92
|
+
- 0 false positives in 500 means the real rate is probably under ~0.6% (rule of three), not zero
|
|
93
|
+
- some mailing lists break base64 bodies by appending footers. those cant be decoded.
|
|
94
|
+
- no learning from corrections yet
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
## data
|
|
99
|
+
built and measured on the [SpamAssassin public corpus](https://spamassassin.apache.org/old/publiccorpus/) (`20021010_spam`, `20021010_easy_ham`, `20021010_hard_ham`).
|
|
100
|
+
to reproduce: extract into `data/spam`, `data/easy_ham`, `data/hard_ham`, then `python -m fivefifty.evaluate` from the repo root.
|
|
101
|
+
|
|
102
|
+
## license
|
|
103
|
+
MIT
|
|
File without changes
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import math
|
|
2
|
+
from fivefifty.parse import subject_and_body
|
|
3
|
+
from fivefifty.tokenizer import tokenize
|
|
4
|
+
from fivefifty.split import load_dataset
|
|
5
|
+
from fivefifty.train import count_words
|
|
6
|
+
|
|
7
|
+
def base_score(prior, fractions):
|
|
8
|
+
score = math.log(prior)
|
|
9
|
+
for (word, fraction) in fractions.items():
|
|
10
|
+
score += math.log(1 - fraction)
|
|
11
|
+
return score
|
|
12
|
+
|
|
13
|
+
def fast_score(base, fractions, words):
|
|
14
|
+
score = base
|
|
15
|
+
for word in words:
|
|
16
|
+
if word in fractions:
|
|
17
|
+
fraction = fractions[word]
|
|
18
|
+
score += math.log(fraction) - math.log(1 - fraction)
|
|
19
|
+
return score
|
|
20
|
+
|
|
21
|
+
def classify(raw_text, counts, email_counts):
|
|
22
|
+
subjected = subject_and_body(raw_text)
|
|
23
|
+
tokenized = tokenize(subjected)
|
|
24
|
+
setted = set(tokenized)
|
|
25
|
+
train_email_total = sum(email_counts.values())
|
|
26
|
+
scores = dict()
|
|
27
|
+
for label in ["spam", "ham"]:
|
|
28
|
+
score = math.log(email_counts[label] / train_email_total )
|
|
29
|
+
for word in setted:
|
|
30
|
+
if word not in counts["spam"] and word not in counts["ham"]:
|
|
31
|
+
continue
|
|
32
|
+
label_counts = counts[label]
|
|
33
|
+
word_count = label_counts.get(word, 0)
|
|
34
|
+
fraction = (word_count + 1) / (email_counts[label] + 2)
|
|
35
|
+
score += math.log(fraction)
|
|
36
|
+
scores[label] = score
|
|
37
|
+
if scores["spam"] > scores["ham"]:
|
|
38
|
+
return "spam", scores
|
|
39
|
+
else:
|
|
40
|
+
return "ham", scores
|
|
41
|
+
|
|
42
|
+
def make_bases(fractions, email_counts):
|
|
43
|
+
train_email_total = sum(email_counts.values())
|
|
44
|
+
bases = {}
|
|
45
|
+
for label in ["spam", "ham"]:
|
|
46
|
+
prior = (email_counts[label] / train_email_total)
|
|
47
|
+
base = base_score(prior, fractions[label])
|
|
48
|
+
bases[label] = base
|
|
49
|
+
return bases
|
|
50
|
+
|
|
51
|
+
def bernoulli_classify(raw_text, fractions, bases):
|
|
52
|
+
subjected = subject_and_body(raw_text)
|
|
53
|
+
tokenized = tokenize(subjected)
|
|
54
|
+
setted = set(tokenized)
|
|
55
|
+
scores = {}
|
|
56
|
+
for label in ["spam", "ham"]:
|
|
57
|
+
scores[label] = fast_score(bases[label], fractions[label], setted)
|
|
58
|
+
if scores["spam"] > scores["ham"]:
|
|
59
|
+
return "spam", scores
|
|
60
|
+
else:
|
|
61
|
+
return "ham", scores
|
|
62
|
+
|
|
63
|
+
def multinomial_classify(raw_text, fractions, email_counts):
|
|
64
|
+
subjected = subject_and_body(raw_text)
|
|
65
|
+
tokenized = tokenize(subjected)
|
|
66
|
+
train_email_total = sum(email_counts.values())
|
|
67
|
+
scores = dict()
|
|
68
|
+
for label in ["spam", "ham"]:
|
|
69
|
+
score = math.log(email_counts[label] / train_email_total)
|
|
70
|
+
for word in tokenized:
|
|
71
|
+
if word in fractions[label]:
|
|
72
|
+
score += math.log(fractions[label][word])
|
|
73
|
+
scores[label] = score
|
|
74
|
+
if scores["spam"] > scores["ham"]:
|
|
75
|
+
return "spam", scores
|
|
76
|
+
else:
|
|
77
|
+
return "ham", scores
|
|
78
|
+
|
|
79
|
+
def explain(raw_text, fractions):
|
|
80
|
+
subjected = subject_and_body(raw_text)
|
|
81
|
+
tokenized = tokenize(subjected)
|
|
82
|
+
pushes = dict()
|
|
83
|
+
for token in tokenized:
|
|
84
|
+
if token not in fractions["spam"]:
|
|
85
|
+
continue
|
|
86
|
+
push = math.log(fractions["spam"][token]) - math.log(fractions["ham"][token])
|
|
87
|
+
pushes[token] = pushes.get(token, 0) + push
|
|
88
|
+
return pushes
|
|
89
|
+
if __name__ == "__main__":
|
|
90
|
+
train, test = load_dataset(0.2, 42)
|
|
91
|
+
counts, email_counts = count_words(train)
|
|
92
|
+
for text, label in test[:10]:
|
|
93
|
+
classification, score = classify(text, counts, email_counts)
|
|
94
|
+
print(f"Real label: {label}\nPrediction: {classification}\nScore: {score}\n")
|
|
95
|
+
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import argparse
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from fivefifty.classify import multinomial_classify, explain
|
|
6
|
+
from fivefifty.model import load_model, train_model, save_model
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def check(text, model):
|
|
10
|
+
|
|
11
|
+
classification = multinomial_classify(text, model["fractions"], model["email_counts"])
|
|
12
|
+
return classification[0].upper()
|
|
13
|
+
|
|
14
|
+
def main():
|
|
15
|
+
|
|
16
|
+
parser = argparse.ArgumentParser(prog="fivefifty", description="A spam filter you can read, that tells you why, and that learns from your own inbox")
|
|
17
|
+
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
18
|
+
check_parser = subparsers.add_parser("check", help="classify one email and show why")
|
|
19
|
+
train_parser = subparsers.add_parser("train", help="train your own model on ham and spam")
|
|
20
|
+
train_parser.add_argument("spam")
|
|
21
|
+
train_parser.add_argument("ham")
|
|
22
|
+
train_parser.add_argument("--out", default="my_model.json")
|
|
23
|
+
check_parser.add_argument("email")
|
|
24
|
+
check_parser.add_argument("--model")
|
|
25
|
+
args = parser.parse_args()
|
|
26
|
+
if args.command == "train":
|
|
27
|
+
model = train_model(args.spam, args.ham)
|
|
28
|
+
save_model(model, args.out)
|
|
29
|
+
print(f"Trained on {model['email_counts']['spam']} spam and {model['email_counts']['ham']} ham. Saved to {args.out}")
|
|
30
|
+
else:
|
|
31
|
+
try:
|
|
32
|
+
with open(args.email, "r", encoding="utf-8", errors="replace") as f:
|
|
33
|
+
text = f.read()
|
|
34
|
+
except FileNotFoundError:
|
|
35
|
+
print(f"{args.email} not found. Check for typos")
|
|
36
|
+
sys.exit(1)
|
|
37
|
+
if args.model:
|
|
38
|
+
model_path = args.model
|
|
39
|
+
else:
|
|
40
|
+
model_path = Path(__file__).parent / "model.json"
|
|
41
|
+
try:
|
|
42
|
+
model = load_model(model_path)
|
|
43
|
+
except FileNotFoundError:
|
|
44
|
+
print(f"{model_path} not found. Check for typos.")
|
|
45
|
+
verdict = check(text, model)
|
|
46
|
+
print(verdict)
|
|
47
|
+
explanation = explain(text, model['fractions'])
|
|
48
|
+
ranked = sorted(explanation.items(), key=lambda pair: pair[1])
|
|
49
|
+
if verdict == "SPAM":
|
|
50
|
+
reasons = ranked[-5:][::-1]
|
|
51
|
+
else:
|
|
52
|
+
reasons = ranked[:5]
|
|
53
|
+
print("="*20,"REASONING", "="*20)
|
|
54
|
+
for word, push in reasons:
|
|
55
|
+
print(f"{word:<10}:\t{push:+.1f}")
|
|
56
|
+
|
|
57
|
+
if __name__ == "__main__":
|
|
58
|
+
main()
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
from fivefifty.classify import classify, load_dataset, bernoulli_classify, make_bases, multinomial_classify
|
|
2
|
+
from fivefifty.train import count_words, make_fractions, count_occurrences, make_multi_fractions
|
|
3
|
+
|
|
4
|
+
def evaluate(test_list, classifier, model_a, model_b):
|
|
5
|
+
results = {"caught": 0, "missed": 0, "false positive": 0, "correct ham": 0}
|
|
6
|
+
classify = classifier
|
|
7
|
+
for text, label in test_list:
|
|
8
|
+
prediction, _ = classify(text, model_a, model_b)
|
|
9
|
+
if prediction == "spam" and label == "spam":
|
|
10
|
+
results["caught"] += 1
|
|
11
|
+
elif prediction == "ham" and label == "spam":
|
|
12
|
+
results["missed"] += 1
|
|
13
|
+
elif prediction == "spam" and label == "ham":
|
|
14
|
+
results["false positive"] += 1
|
|
15
|
+
else:
|
|
16
|
+
results["correct ham"] += 1
|
|
17
|
+
|
|
18
|
+
return results
|
|
19
|
+
|
|
20
|
+
def metrics(results):
|
|
21
|
+
total_unsummed = results.values()
|
|
22
|
+
total = sum(total_unsummed)
|
|
23
|
+
accuracy = (results["caught"] + results["correct ham"]) / total
|
|
24
|
+
precision_bottom = results["caught"] + results["false positive"]
|
|
25
|
+
if precision_bottom == 0:
|
|
26
|
+
precision = 0
|
|
27
|
+
else:
|
|
28
|
+
precision = results["caught"] / precision_bottom
|
|
29
|
+
recall_bottom = results["caught"] + results["missed"]
|
|
30
|
+
if recall_bottom == 0:
|
|
31
|
+
recall = 0
|
|
32
|
+
else:
|
|
33
|
+
recall = results["caught"] / recall_bottom
|
|
34
|
+
return accuracy, precision, recall
|
|
35
|
+
if __name__ == "__main__":
|
|
36
|
+
train, test = load_dataset(0.2, 42)
|
|
37
|
+
counts, email_counts = count_words(train)
|
|
38
|
+
results = evaluate(test, classify, counts, email_counts)
|
|
39
|
+
accuracy, precision, recall = metrics(results)
|
|
40
|
+
print(f"{results}, {accuracy:.1%}, {precision:.1%}, {recall:.1%}")
|
|
41
|
+
print("-"*40)
|
|
42
|
+
fractions = make_fractions(counts, email_counts)
|
|
43
|
+
bases = make_bases(fractions, email_counts)
|
|
44
|
+
res_ber = evaluate(test, bernoulli_classify, fractions, bases)
|
|
45
|
+
accuracy2, precision2, recall2 = metrics(res_ber)
|
|
46
|
+
print(f"{res_ber}, {accuracy2:.1%}, {precision2:.1%}, {recall2:.1%}")
|
|
47
|
+
print("-"*40)
|
|
48
|
+
final_fractions = make_fractions(counts, email_counts, 112)
|
|
49
|
+
final_bases = make_bases(final_fractions, email_counts)
|
|
50
|
+
res_final = evaluate(test, bernoulli_classify, final_fractions, final_bases)
|
|
51
|
+
accuracy3, precision3, recall3 = metrics(res_final)
|
|
52
|
+
print(f"{res_final}, {accuracy3:.1%}, {precision3:.1%}, {recall3:.1%}")
|
|
53
|
+
print("-"*40)
|
|
54
|
+
multi_counts, multi_email_counts, multi_word_totals = count_occurrences(train)
|
|
55
|
+
multi_fractions = make_multi_fractions(multi_counts, multi_word_totals)
|
|
56
|
+
res_multi = evaluate(test, multinomial_classify, multi_fractions, multi_email_counts,)
|
|
57
|
+
accuracy4, precision4, recall4 = metrics(res_multi)
|
|
58
|
+
print(f"multinomial, {res_multi}, {accuracy4:.1%}, {precision4:.1%}, {recall4:.1%}")
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
import math
|
|
2
|
+
import email
|
|
3
|
+
from fivefifty.loader import load_folder
|
|
4
|
+
from fivefifty.model import load_model
|
|
5
|
+
from fivefifty.parse import subject_and_body
|
|
6
|
+
from fivefifty.tokenizer import tokenize
|
|
7
|
+
from fivefifty.classify import load_dataset, count_words, base_score, fast_score, make_bases, bernoulli_classify, \
|
|
8
|
+
multinomial_classify, explain
|
|
9
|
+
from fivefifty.train import make_fractions, count_occurrences, make_multi_fractions
|
|
10
|
+
def lean(spam_count, ham_count, spam_mails, ham_mails):
|
|
11
|
+
spam_fraction = (spam_count + 1) / (spam_mails + 2)
|
|
12
|
+
ham_fraction = (ham_count + 1) / (ham_mails + 2)
|
|
13
|
+
return math.log(spam_fraction) - math.log(ham_fraction)
|
|
14
|
+
from fivefifty.split import train_test_split
|
|
15
|
+
from fivefifty.evaluate import evaluate, metrics
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def bernoulli_score(prior, fractions, email_words):
|
|
20
|
+
score = math.log(prior)
|
|
21
|
+
for (word, fraction) in fractions.items():
|
|
22
|
+
if word in email_words:
|
|
23
|
+
score += math.log(fraction)
|
|
24
|
+
else:
|
|
25
|
+
score += math.log(1-fraction)
|
|
26
|
+
return score
|
|
27
|
+
|
|
28
|
+
def count_base64(emails):
|
|
29
|
+
counter = 0
|
|
30
|
+
for text, label in emails:
|
|
31
|
+
if "content-transfer-encoding: base64" in text.lower():
|
|
32
|
+
counter += 1
|
|
33
|
+
return counter
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
if __name__ == "__main__":
|
|
37
|
+
# for n in range(8):
|
|
38
|
+
# lean_result = lean(0, n, 400, 2000)
|
|
39
|
+
# if lean_result > 0:
|
|
40
|
+
# classification = "Spam"
|
|
41
|
+
# else:
|
|
42
|
+
# classification = "Ham"
|
|
43
|
+
# print(n, f"{lean_result:.3f}\nClassification: {classification}")
|
|
44
|
+
# print("="*40)
|
|
45
|
+
# spam_fractions = {"click": 0.51, "free": 0.48, "patch": 0.01}
|
|
46
|
+
# ham_fractions = {"click": 0.17, "free": 0.13, "patch": 0.02}
|
|
47
|
+
# email_words = set({"click", "free"})
|
|
48
|
+
# x = bernoulli_score(400/2400, spam_fractions, email_words)
|
|
49
|
+
# y = bernoulli_score(2000/2400, ham_fractions, email_words)
|
|
50
|
+
# print(f"X = {x}, Y = {y}")
|
|
51
|
+
# if x > y:
|
|
52
|
+
# print("x wins!")
|
|
53
|
+
# else:
|
|
54
|
+
# print("Y wins!")
|
|
55
|
+
# print("="*40)
|
|
56
|
+
# spam_base = base_score(400/2400, spam_fractions)
|
|
57
|
+
# ham_base = base_score(2000/2400, ham_fractions)
|
|
58
|
+
# fast_spam = fast_score(spam_base, spam_fractions, email_words)
|
|
59
|
+
# fast_ham = fast_score(ham_base, ham_fractions, email_words)
|
|
60
|
+
# print(f"X: {fast_spam}\nY: {fast_ham}")
|
|
61
|
+
# print("="*40)
|
|
62
|
+
# train, test = load_dataset(0.2, 42)
|
|
63
|
+
# counts, email_counts = count_words(train)
|
|
64
|
+
# fractions = make_fractions(counts, email_counts)
|
|
65
|
+
# print (f"Spam: {len(fractions['spam'])}\nHam: {len(fractions['ham'])}.")
|
|
66
|
+
# for word in ("click", "free", "patch"):
|
|
67
|
+
# print(
|
|
68
|
+
# f"{word}, spam: {fractions['spam'][word]:.1%}, ham: {fractions['ham'][word]:.1%}")
|
|
69
|
+
# train_email_total = sum(email_counts.values())
|
|
70
|
+
# spam_prior = (email_counts['spam'] / train_email_total)
|
|
71
|
+
# spam_base_result = base_score(spam_prior, fractions['spam'])
|
|
72
|
+
# ham_prior = (email_counts['ham'] / train_email_total)
|
|
73
|
+
# ham_base_result = base_score(ham_prior, fractions['ham'])
|
|
74
|
+
# print('-'*40)
|
|
75
|
+
# print(f'Spam base result: {spam_base_result}\nHam Base result: {ham_base_result}')
|
|
76
|
+
# for text, label in test[:1]:
|
|
77
|
+
# subjected = subject_and_body(text)
|
|
78
|
+
# tokenized = tokenize(subjected)
|
|
79
|
+
# setted = set(tokenized)
|
|
80
|
+
# print(f"{label}, uniq words: {len(setted)}")
|
|
81
|
+
# spam_slow = bernoulli_score(spam_prior, fractions['spam'], setted)
|
|
82
|
+
# spam_fast = fast_score(spam_base_result, fractions['spam'], setted)
|
|
83
|
+
# print('-'*40)
|
|
84
|
+
# print(f'spam_slow: {spam_slow}, spam_fast: {spam_fast}')
|
|
85
|
+
# ham_slow = bernoulli_score(ham_prior, fractions['ham'], setted)
|
|
86
|
+
# ham_fast = fast_score(ham_base_result, fractions['ham'], setted)
|
|
87
|
+
# print('-'*40)
|
|
88
|
+
# print(f"ham_slow: {ham_slow}, ham_fast: {ham_fast}")
|
|
89
|
+
# print("-"*40)
|
|
90
|
+
# bases = make_bases(fractions,email_counts)
|
|
91
|
+
# bern_res = bernoulli_classify(text, fractions, bases)
|
|
92
|
+
# print("-"*40)
|
|
93
|
+
# print(bern_res)
|
|
94
|
+
# only_one = 0
|
|
95
|
+
# only_one_email_words = list()
|
|
96
|
+
# for word in fractions['spam']:
|
|
97
|
+
# total = counts['ham'].get(word, 0) + counts['spam'].get(word, 0)
|
|
98
|
+
# if total == 1:
|
|
99
|
+
# only_one_email_words.append(word)
|
|
100
|
+
# only_one += 1
|
|
101
|
+
# print(only_one)
|
|
102
|
+
# print(only_one_email_words[:30])
|
|
103
|
+
# fit, val = train_test_split(train, 0.2, 42,)
|
|
104
|
+
# print(len(fit), len(val))
|
|
105
|
+
# fractions2 = make_fractions(counts, email_counts, 2)
|
|
106
|
+
# print(len(fractions2['spam']), len(fractions2['ham']))
|
|
107
|
+
# fit_counts, fit_email_counts = count_words(fit)
|
|
108
|
+
# # print(fit_email_counts)
|
|
109
|
+
# ###### TRYING SEVERAL N
|
|
110
|
+
# print("-"*40)
|
|
111
|
+
# for n in [1, 2, 5, 10, 20, 50, 90]:
|
|
112
|
+
# fit_fractions = make_fractions(fit_counts, fit_email_counts, n)
|
|
113
|
+
# fit_bases = make_bases(fit_fractions, fit_email_counts)
|
|
114
|
+
# evaluation = evaluate(val, bernoulli_classify, fit_fractions, fit_bases)
|
|
115
|
+
# accuracy, precision, recall = metrics(evaluation)
|
|
116
|
+
# print(f"{n}, {evaluation}, precision: {precision:.1%}, recall: {recall:.1%}, vocab size: {len(fit_fractions['spam'])}")
|
|
117
|
+
# print("-"*40)
|
|
118
|
+
# counts,_,word_totals = count_occurrences(fit)
|
|
119
|
+
# print(word_totals)
|
|
120
|
+
# print(counts['spam'].get('free', 0))
|
|
121
|
+
# print("-"*40)
|
|
122
|
+
# multi_counts, multi_email_counts, multi_word_totals = count_occurrences(fit)
|
|
123
|
+
# multi_fractions = make_multi_fractions(multi_counts, multi_word_totals)
|
|
124
|
+
# multi_evaluation = evaluate(val, multinomial_classify, multi_fractions, multi_email_counts)
|
|
125
|
+
# multi_accuracy, multi_precision, multi_recall = metrics(multi_evaluation)
|
|
126
|
+
# print(
|
|
127
|
+
# f"multinomial, {multi_evaluation}, precision: {multi_precision:.1%}, recall: {multi_recall:.1%}, vocab size: {len(multi_fractions)}")
|
|
128
|
+
# spam = load_folder("./data/spam", "spam")
|
|
129
|
+
# ez_ham = load_folder("./data/easy_ham", "ham")
|
|
130
|
+
# hard_ham = load_folder("./data/hard_ham", "hard_ham")
|
|
131
|
+
# count_spam_result = count_base64(spam) / len(spam)
|
|
132
|
+
# count_ez_ham_result = count_base64(ez_ham) / len(ez_ham)
|
|
133
|
+
# count_hard_ham_result = count_base64(hard_ham) / len(hard_ham)
|
|
134
|
+
# print(f"Spam: {count_spam_result:.1%}\nez_ham: {count_ez_ham_result:.1%}\nHard_ham: {count_hard_ham_result:.1%}\n")
|
|
135
|
+
# for text, label in val:
|
|
136
|
+
# prediction, _ = multinomial_classify(text, multi_fractions, multi_email_counts)
|
|
137
|
+
# if label == "spam" and prediction == "ham":
|
|
138
|
+
# print(f"Base64: {'content-transfer-encoding: base64' in text.lower()}")
|
|
139
|
+
# print(text[:200])
|
|
140
|
+
# stored = ""
|
|
141
|
+
# for text, label in val:
|
|
142
|
+
# if label == "spam" and 'content-transfer-encoding: base64' in text.lower():
|
|
143
|
+
# stored = text
|
|
144
|
+
# break
|
|
145
|
+
# msg = email.message_from_string(stored)
|
|
146
|
+
# print(msg.is_multipart(), msg.get_content_type())
|
|
147
|
+
# for part in msg.walk():
|
|
148
|
+
# print(part.get_content_type(), part.get("Content-Transfer-Encoding"))
|
|
149
|
+
# payload = part.get_payload(decode=True)
|
|
150
|
+
# charset = part.get_content_charset()
|
|
151
|
+
# if charset is None:
|
|
152
|
+
# decoded = payload.decode('latin-1', errors="replace")
|
|
153
|
+
# else:
|
|
154
|
+
# decoded = payload.decode(charset, errors="replace")
|
|
155
|
+
# print(decoded)
|
|
156
|
+
#
|
|
157
|
+
# print(subject_and_body_mime(stored)[:300])
|
|
158
|
+
# plain = ""
|
|
159
|
+
# for text, label in val:
|
|
160
|
+
# if label == "spam" and 'contet-transfer-encoding: base64' not in text.lower():
|
|
161
|
+
# plain = text
|
|
162
|
+
# break
|
|
163
|
+
# print(subject_and_body(plain)[:300])
|
|
164
|
+
# print(subject_and_body_mime(plain)[:300])
|
|
165
|
+
model = load_model("fivefifty/model.json")
|
|
166
|
+
with open("data/real/testspam.eml", encoding="utf-8", errors="replace") as f:
|
|
167
|
+
test = f.read()
|
|
168
|
+
pushes = explain(test, model['fractions'])
|
|
169
|
+
ranked = sorted(pushes.items(), key=lambda pair: pair[1])
|
|
170
|
+
print(f"Spammiest: {ranked[-5:]}\nHammiest: {ranked[:5]}")
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
from pydoc import replace
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def load_folder(path, label):
|
|
6
|
+
path_loaded = Path(path)
|
|
7
|
+
results = list()
|
|
8
|
+
for entry in path_loaded.iterdir():
|
|
9
|
+
entry_string = entry.read_text(encoding="utf-8", errors="replace")
|
|
10
|
+
results.append((entry_string, label))
|
|
11
|
+
return results
|
|
12
|
+
|
|
13
|
+
def count_html(emails):
|
|
14
|
+
counter = 0
|
|
15
|
+
for text, label in emails:
|
|
16
|
+
if "<html" in text.lower():
|
|
17
|
+
counter += 1
|
|
18
|
+
return counter
|
|
19
|
+
|
|
20
|
+
if __name__ == "__main__":
|
|
21
|
+
spam = load_folder("./data/spam", "spam")
|
|
22
|
+
ez_ham = load_folder("./data/easy_ham", "ham")
|
|
23
|
+
hard_ham = load_folder("./data/hard_ham", "hard_ham")
|
|
24
|
+
s_results = (count_html(spam) / len(spam))
|
|
25
|
+
e_results = (count_html(ez_ham) / len(ez_ham))
|
|
26
|
+
h_results = (count_html(hard_ham)/ len(hard_ham))
|
|
27
|
+
print(f"s_results: {s_results:%}, ez_ham: {e_results:%}, hard_ham: {h_results:%}")
|
|
28
|
+
|