email-reply-extractor 1.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- email_reply_extractor-1.1.0/LICENSE +21 -0
- email_reply_extractor-1.1.0/PKG-INFO +243 -0
- email_reply_extractor-1.1.0/README.md +220 -0
- email_reply_extractor-1.1.0/pyproject.toml +50 -0
- email_reply_extractor-1.1.0/setup.cfg +4 -0
- email_reply_extractor-1.1.0/src/email_reply_extractor/__init__.py +49 -0
- email_reply_extractor-1.1.0/src/email_reply_extractor/_fragments.py +130 -0
- email_reply_extractor-1.1.0/src/email_reply_extractor/cleaning.py +601 -0
- email_reply_extractor-1.1.0/src/email_reply_extractor/equivalence.py +43 -0
- email_reply_extractor-1.1.0/src/email_reply_extractor/extraction.py +1007 -0
- email_reply_extractor-1.1.0/src/email_reply_extractor/html_text.py +427 -0
- email_reply_extractor-1.1.0/src/email_reply_extractor/py.typed +0 -0
- email_reply_extractor-1.1.0/src/email_reply_extractor.egg-info/PKG-INFO +243 -0
- email_reply_extractor-1.1.0/src/email_reply_extractor.egg-info/SOURCES.txt +20 -0
- email_reply_extractor-1.1.0/src/email_reply_extractor.egg-info/dependency_links.txt +1 -0
- email_reply_extractor-1.1.0/src/email_reply_extractor.egg-info/requires.txt +4 -0
- email_reply_extractor-1.1.0/src/email_reply_extractor.egg-info/top_level.txt +1 -0
- email_reply_extractor-1.1.0/tests/test_cleaning.py +490 -0
- email_reply_extractor-1.1.0/tests/test_equivalence.py +30 -0
- email_reply_extractor-1.1.0/tests/test_extraction.py +962 -0
- email_reply_extractor-1.1.0/tests/test_extraction_version.py +93 -0
- email_reply_extractor-1.1.0/tests/test_html_text.py +234 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Jay Daley
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: email-reply-extractor
|
|
3
|
+
Version: 1.1.0
|
|
4
|
+
Summary: Extract the author's newly written text from an email: quoted-text removal plus scoring-ready cleaning
|
|
5
|
+
Author: Jay Daley
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/JayDaley/email-reply-extractor
|
|
8
|
+
Project-URL: Changelog, https://github.com/JayDaley/email-reply-extractor/blob/main/CHANGELOG.md
|
|
9
|
+
Keywords: email,reply,quoting,signature,mailing-list,text-extraction
|
|
10
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Topic :: Communications :: Email
|
|
15
|
+
Classifier: Topic :: Text Processing
|
|
16
|
+
Requires-Python: >=3.10
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Provides-Extra: dev
|
|
20
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
21
|
+
Requires-Dist: ruff>=0.6; extra == "dev"
|
|
22
|
+
Dynamic: license-file
|
|
23
|
+
|
|
24
|
+
# email-reply-extractor
|
|
25
|
+
|
|
26
|
+
Recover the author's newly written text from an email — typically mailing-list
|
|
27
|
+
mail — in two stages. Pure standard library, no runtime dependencies,
|
|
28
|
+
Python 3.10 or later.
|
|
29
|
+
|
|
30
|
+
## What it does
|
|
31
|
+
|
|
32
|
+
Stage 1, `extract_new_text`, removes everything the author did not write in
|
|
33
|
+
this message: quoted lines (any indentation, including the `...>` form that
|
|
34
|
+
survives `format=flowed` rewrapping), attribution lines ("On … wrote:", the
|
|
35
|
+
German "Am … schrieb", the Chinese Outlook and Alibaba Mail forms),
|
|
36
|
+
forwarded/quote-header blocks, "Original message" dividers, the quoted thread
|
|
37
|
+
that follows a sign-off boundary, and — when the caller supplies them —
|
|
38
|
+
content provably present in the thread parent or in the HTML part's
|
|
39
|
+
blockquotes. It **keeps** the author's greetings, sign-offs and signature
|
|
40
|
+
blocks.
|
|
41
|
+
|
|
42
|
+
Stage 2, `clean_for_scoring`, removes that formulaic furniture from stage-1
|
|
43
|
+
text: greetings, sign-offs, signature blocks, mailing-list footers, mobile
|
|
44
|
+
taglines and trailing legal disclaimers. It also reports which non-blank input
|
|
45
|
+
lines it removed.
|
|
46
|
+
|
|
47
|
+
The split is deliberate. The stage-1 text is the author's full novel content,
|
|
48
|
+
which is what an application should store and display; the stage-2 text is a
|
|
49
|
+
strict, documented subset of it, which is what an application should feed to a
|
|
50
|
+
downstream consumer such as a classifier. Because stage 2 reports the lines it
|
|
51
|
+
dropped, a stage-1 view can mark exactly what was set aside.
|
|
52
|
+
|
|
53
|
+
Both stages are pure and I/O-free. `extract_new_text` never raises: an
|
|
54
|
+
unexpected error is reported as `status="failed"` so a single bad message
|
|
55
|
+
cannot stall a pipeline.
|
|
56
|
+
|
|
57
|
+
The rules of both stages are documented in [docs/extraction.md](docs/extraction.md).
|
|
58
|
+
|
|
59
|
+
## Install
|
|
60
|
+
|
|
61
|
+
Not yet published on PyPI. Install from the repository:
|
|
62
|
+
|
|
63
|
+
```
|
|
64
|
+
pip install git+https://github.com/JayDaley/email-reply-extractor
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
## Quickstart
|
|
68
|
+
|
|
69
|
+
```python
|
|
70
|
+
from email_reply_extractor import clean_for_scoring, extract_new_text
|
|
71
|
+
|
|
72
|
+
body = """Hi Dan,
|
|
73
|
+
|
|
74
|
+
Two comments below.
|
|
75
|
+
|
|
76
|
+
On Tue, Jul 7, 2026, at 03:25, Dan Wing wrote:
|
|
77
|
+
> I would lean heavily towards solution (2).
|
|
78
|
+
|
|
79
|
+
Agreed, and the draft already says so.
|
|
80
|
+
|
|
81
|
+
Best,
|
|
82
|
+
Lucas
|
|
83
|
+
"""
|
|
84
|
+
|
|
85
|
+
result = extract_new_text(body)
|
|
86
|
+
result.status # 'ok'
|
|
87
|
+
result.method # 'erp'
|
|
88
|
+
result.text # greeting + both prose lines + the "Best, / Lucas" sign-off
|
|
89
|
+
|
|
90
|
+
cleaned = clean_for_scoring(result.text)
|
|
91
|
+
cleaned.text # 'Two comments below.\nAgreed, and the draft already says so.'
|
|
92
|
+
cleaned.ignored_lines # [0, 5, 6] — the greeting and the two sign-off lines
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
The quote and its attribution line are gone from `result.text`; the greeting
|
|
96
|
+
and the sign-off are still there, and `clean_for_scoring` is what removes them.
|
|
97
|
+
|
|
98
|
+
Two optional arguments improve the result when the caller can supply them, and
|
|
99
|
+
change nothing when they are omitted:
|
|
100
|
+
|
|
101
|
+
```python
|
|
102
|
+
# The thread parent's raw body, resolved by the caller from In-Reply-To.
|
|
103
|
+
# Removes a quoted previous message that carries no '>' markers at all.
|
|
104
|
+
result = extract_new_text(body, parent_body=parent_body)
|
|
105
|
+
|
|
106
|
+
# The message's decoded text/html part. Used as a structural oracle: as the
|
|
107
|
+
# body when there is no usable plain part, as a replacement for a flattened
|
|
108
|
+
# plain part, or to remove a quoted message leaked unmarked into the plain part.
|
|
109
|
+
result = extract_new_text(body, html_body=html_body)
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
`clean_for_scoring` takes the HTML signature container's visible text as an
|
|
113
|
+
optional hint, which removes signature furniture that carries no `-- `
|
|
114
|
+
delimiter and no recognizable contact shape:
|
|
115
|
+
|
|
116
|
+
```python
|
|
117
|
+
from email_reply_extractor import split_html_parts
|
|
118
|
+
|
|
119
|
+
parts = split_html_parts(html_body)
|
|
120
|
+
cleaned = clean_for_scoring(result.text, html_signature_text=parts.signature_text)
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
## Public API
|
|
124
|
+
|
|
125
|
+
| Name | What it is |
|
|
126
|
+
|---|---|
|
|
127
|
+
| `extract_new_text(body, parent_body=None, html_body=None)` | Stage 1; returns an `ExtractionResult`. |
|
|
128
|
+
| `ExtractionResult` | Frozen dataclass: `text`, `method`, `status`. |
|
|
129
|
+
| `STATUS_OK`, `STATUS_EMPTY`, `STATUS_FAILED` | The three `status` values. |
|
|
130
|
+
| `EXTRACTION_VERSION` | `int` generation stamp of the text-deriving routine. |
|
|
131
|
+
| `clean_for_scoring(extracted_text, html_signature_text=None)` | Stage 2; returns a `CleanResult`. |
|
|
132
|
+
| `CleanResult` | Frozen dataclass: `text`, `ignored_lines`. |
|
|
133
|
+
| `html_to_text(html)` | Visible text of an HTML part, block structure as newlines. |
|
|
134
|
+
| `split_html_parts(html)` | The same text split into `HtmlParts(novel_text, quoted_text, signature_text)`. |
|
|
135
|
+
| `HtmlParts` | Frozen dataclass returned by `split_html_parts`. |
|
|
136
|
+
| `strip_parent_content(text, parent_text)` | Remove lines of `text` provably drawn from `parent_text`. |
|
|
137
|
+
| `__version__` | The package version. |
|
|
138
|
+
|
|
139
|
+
### Status values
|
|
140
|
+
|
|
141
|
+
- `ok` — text was extracted.
|
|
142
|
+
- `empty` — nothing was left: a blank body, an HTML-only message with no usable
|
|
143
|
+
HTML text, or a body that reduced to quotes alone.
|
|
144
|
+
- `failed` — extraction raised; `text` is empty.
|
|
145
|
+
|
|
146
|
+
### Method strings
|
|
147
|
+
|
|
148
|
+
`ExtractionResult.method` records which path produced the text, so a result is
|
|
149
|
+
auditable after the fact. The base value is one of:
|
|
150
|
+
|
|
151
|
+
- `none` — no body to work from.
|
|
152
|
+
- `erp` — the vendored fragment scan needed no cleanup.
|
|
153
|
+
- `erp+custom` — the fragment scan plus the custom cleanup pass.
|
|
154
|
+
- `custom-fallback` — the over-strip guard fired, and the custom pass ran on
|
|
155
|
+
the whole body.
|
|
156
|
+
- `failed` — extraction raised.
|
|
157
|
+
|
|
158
|
+
Three modifiers combine with the base value:
|
|
159
|
+
|
|
160
|
+
- `html-` prefix — the text was derived from the HTML part, either because
|
|
161
|
+
there was no usable plain body or because a flattened plain body was
|
|
162
|
+
replaced (for example `html-erp+custom`).
|
|
163
|
+
- `+parent-diff` suffix — the parent-diff assist removed content (for example
|
|
164
|
+
`erp+parent-diff`).
|
|
165
|
+
- `+html-quote` suffix — the HTML quoted-text oracle removed content from a
|
|
166
|
+
plain extraction.
|
|
167
|
+
|
|
168
|
+
## Extraction version
|
|
169
|
+
|
|
170
|
+
`EXTRACTION_VERSION` is an `int` identifying the generation of the routine that
|
|
171
|
+
derives the text. It is independent of the package version, only ever
|
|
172
|
+
increases, and is incremented whenever a change to the extraction, cleaning,
|
|
173
|
+
HTML or vendored-scanner modules can alter the derived text. It is currently
|
|
174
|
+
**5**.
|
|
175
|
+
|
|
176
|
+
An application that stores derived text should store `EXTRACTION_VERSION`
|
|
177
|
+
alongside it and compare the stored stamp with the running constant using `<`
|
|
178
|
+
to decide whether the text needs re-deriving. The full contract, including the
|
|
179
|
+
digest that pins the behavior, is in [docs/versioning.md](docs/versioning.md).
|
|
180
|
+
Generation 5 (shipped in v1.1.0) and the work deferred from it are described in
|
|
181
|
+
[docs/generation-5.md](docs/generation-5.md). Because a generation bump can
|
|
182
|
+
move bytes without changing substance, the package exports `tolerant_lines`
|
|
183
|
+
and `texts_equivalent` so a consumer can re-derive stored text and keep
|
|
184
|
+
expensive downstream results (for example a paid classifier score) when only
|
|
185
|
+
whitespace moved.
|
|
186
|
+
|
|
187
|
+
## Supported Python versions
|
|
188
|
+
|
|
189
|
+
Python 3.10 through 3.14; each is exercised in CI.
|
|
190
|
+
|
|
191
|
+
## Comparison with alternatives
|
|
192
|
+
|
|
193
|
+
The two most widely used Python libraries for this task are
|
|
194
|
+
[email-reply-parser](https://github.com/zapier/email-reply-parser) (Zapier's
|
|
195
|
+
port of GitHub's Email Reply Parser) and
|
|
196
|
+
[talon](https://github.com/mailgun/talon) (Mailgun). The table states each
|
|
197
|
+
project's documented behavior; release data is from PyPI as of September 2026.
|
|
198
|
+
|
|
199
|
+
| | email-reply-extractor | email-reply-parser | talon |
|
|
200
|
+
|---|---|---|---|
|
|
201
|
+
| Extracts | The author's full novel content (stage 1), then the substantive prose alone (stage 2) | The latest reply, quoted text and signatures hidden | The reply with quotations removed; signatures separately |
|
|
202
|
+
| Signature handling | Kept in stage 1, removed and reported line-by-line in stage 2 | Hidden along with the reply's quoted fragments; not reported | Regex ("brute force") extractor, plus an SVM machine-learning extractor |
|
|
203
|
+
| HTML part | Used as a structural oracle: as the body when no usable plain part exists, replacing a flattened plain part, or removing a quoted message leaked unmarked into the plain part | Not handled | Quotation extraction has an HTML mode (lxml-based) |
|
|
204
|
+
| Thread parent | Optional parent-diff assist removes a quoted previous message that carries no `>` markers | Not used | Not used |
|
|
205
|
+
| Quote headers | Outlook-style blocks, including German and Chinese label sets, folded and double-spaced renderings | `From:`/`Sent:`/`To:`/`Subject:` fragment detection, English only | Delimiter patterns (e.g. `-----Original Message-----`), several languages |
|
|
206
|
+
| Output stability | Behavior pinned by an integer generation (`EXTRACTION_VERSION`) and a corpus digest test; whitespace-tolerant equivalence helpers for consumers | Not versioned separately | Not versioned separately |
|
|
207
|
+
| Test corpus | Hand-labeled real mailing-list messages (`tests/fixtures`) | Unit-test fixture emails | Unit-test fixture emails; ML training data |
|
|
208
|
+
| Runtime dependencies | None (standard library) | None (standard library) | lxml, regex, numpy, scipy, scikit-learn, chardet, and others |
|
|
209
|
+
| Latest release | 1.1.0 (2026) | 0.5.12 (2020) | 1.4.4 (2017) |
|
|
210
|
+
| License | MIT | MIT | Apache 2.0 |
|
|
211
|
+
|
|
212
|
+
The fragment scanner at this library's core derives from email-reply-parser —
|
|
213
|
+
see Provenance below — so on a plain-text body with English quoting the two
|
|
214
|
+
produce related results; the differences above are what this library adds on
|
|
215
|
+
top of that scan.
|
|
216
|
+
|
|
217
|
+
## Provenance
|
|
218
|
+
|
|
219
|
+
The code was extracted from
|
|
220
|
+
[mailing-list-ai-check](https://github.com/JayDaley/mailing-list-ai-check), an
|
|
221
|
+
application that checks mailing-list mail with an AI detector, where it shipped
|
|
222
|
+
as four extraction generations: 1 (initial release), 2 (from that
|
|
223
|
+
application's v1.2.0), 3 (from v1.11.0) and 4 (from v1.15.0). The hand-labeled
|
|
224
|
+
fixture corpus and its provenance came with it; see
|
|
225
|
+
[tests/fixtures/README.md](tests/fixtures/README.md).
|
|
226
|
+
|
|
227
|
+
The extraction also absorbed the application's only runtime dependency,
|
|
228
|
+
`email-reply-parser` 0.5.12 (MIT, Zapier's Python port of GitHub's Email Reply
|
|
229
|
+
Parser). Its fragment scanner is vendored, trimmed to the scan path this
|
|
230
|
+
package consumes, as `src/email_reply_extractor/_fragments.py`; two upstream
|
|
231
|
+
defects are preserved deliberately so the output does not move (see
|
|
232
|
+
[docs/generation-5.md](docs/generation-5.md)). The package therefore has no
|
|
233
|
+
runtime dependencies.
|
|
234
|
+
|
|
235
|
+
Two pieces of evidence support the claim that v1.0.0 derives the same text as
|
|
236
|
+
the application it came from: the recorded corpus digest carried over
|
|
237
|
+
unchanged, and a differential run over 110,725 real stored messages compared
|
|
238
|
+
the old (`email_reply_parser`) and new (vendored) implementations on stage-1
|
|
239
|
+
text, method and status and on the stage-2 composite, with zero mismatches.
|
|
240
|
+
|
|
241
|
+
## License
|
|
242
|
+
|
|
243
|
+
MIT. See [LICENSE](LICENSE).
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
# email-reply-extractor
|
|
2
|
+
|
|
3
|
+
Recover the author's newly written text from an email — typically mailing-list
|
|
4
|
+
mail — in two stages. Pure standard library, no runtime dependencies,
|
|
5
|
+
Python 3.10 or later.
|
|
6
|
+
|
|
7
|
+
## What it does
|
|
8
|
+
|
|
9
|
+
Stage 1, `extract_new_text`, removes everything the author did not write in
|
|
10
|
+
this message: quoted lines (any indentation, including the `...>` form that
|
|
11
|
+
survives `format=flowed` rewrapping), attribution lines ("On … wrote:", the
|
|
12
|
+
German "Am … schrieb", the Chinese Outlook and Alibaba Mail forms),
|
|
13
|
+
forwarded/quote-header blocks, "Original message" dividers, the quoted thread
|
|
14
|
+
that follows a sign-off boundary, and — when the caller supplies them —
|
|
15
|
+
content provably present in the thread parent or in the HTML part's
|
|
16
|
+
blockquotes. It **keeps** the author's greetings, sign-offs and signature
|
|
17
|
+
blocks.
|
|
18
|
+
|
|
19
|
+
Stage 2, `clean_for_scoring`, removes that formulaic furniture from stage-1
|
|
20
|
+
text: greetings, sign-offs, signature blocks, mailing-list footers, mobile
|
|
21
|
+
taglines and trailing legal disclaimers. It also reports which non-blank input
|
|
22
|
+
lines it removed.
|
|
23
|
+
|
|
24
|
+
The split is deliberate. The stage-1 text is the author's full novel content,
|
|
25
|
+
which is what an application should store and display; the stage-2 text is a
|
|
26
|
+
strict, documented subset of it, which is what an application should feed to a
|
|
27
|
+
downstream consumer such as a classifier. Because stage 2 reports the lines it
|
|
28
|
+
dropped, a stage-1 view can mark exactly what was set aside.
|
|
29
|
+
|
|
30
|
+
Both stages are pure and I/O-free. `extract_new_text` never raises: an
|
|
31
|
+
unexpected error is reported as `status="failed"` so a single bad message
|
|
32
|
+
cannot stall a pipeline.
|
|
33
|
+
|
|
34
|
+
The rules of both stages are documented in [docs/extraction.md](docs/extraction.md).
|
|
35
|
+
|
|
36
|
+
## Install
|
|
37
|
+
|
|
38
|
+
Not yet published on PyPI. Install from the repository:
|
|
39
|
+
|
|
40
|
+
```
|
|
41
|
+
pip install git+https://github.com/JayDaley/email-reply-extractor
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
## Quickstart
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
from email_reply_extractor import clean_for_scoring, extract_new_text
|
|
48
|
+
|
|
49
|
+
body = """Hi Dan,
|
|
50
|
+
|
|
51
|
+
Two comments below.
|
|
52
|
+
|
|
53
|
+
On Tue, Jul 7, 2026, at 03:25, Dan Wing wrote:
|
|
54
|
+
> I would lean heavily towards solution (2).
|
|
55
|
+
|
|
56
|
+
Agreed, and the draft already says so.
|
|
57
|
+
|
|
58
|
+
Best,
|
|
59
|
+
Lucas
|
|
60
|
+
"""
|
|
61
|
+
|
|
62
|
+
result = extract_new_text(body)
|
|
63
|
+
result.status # 'ok'
|
|
64
|
+
result.method # 'erp'
|
|
65
|
+
result.text # greeting + both prose lines + the "Best, / Lucas" sign-off
|
|
66
|
+
|
|
67
|
+
cleaned = clean_for_scoring(result.text)
|
|
68
|
+
cleaned.text # 'Two comments below.\nAgreed, and the draft already says so.'
|
|
69
|
+
cleaned.ignored_lines # [0, 5, 6] — the greeting and the two sign-off lines
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
The quote and its attribution line are gone from `result.text`; the greeting
|
|
73
|
+
and the sign-off are still there, and `clean_for_scoring` is what removes them.
|
|
74
|
+
|
|
75
|
+
Two optional arguments improve the result when the caller can supply them, and
|
|
76
|
+
change nothing when they are omitted:
|
|
77
|
+
|
|
78
|
+
```python
|
|
79
|
+
# The thread parent's raw body, resolved by the caller from In-Reply-To.
|
|
80
|
+
# Removes a quoted previous message that carries no '>' markers at all.
|
|
81
|
+
result = extract_new_text(body, parent_body=parent_body)
|
|
82
|
+
|
|
83
|
+
# The message's decoded text/html part. Used as a structural oracle: as the
|
|
84
|
+
# body when there is no usable plain part, as a replacement for a flattened
|
|
85
|
+
# plain part, or to remove a quoted message leaked unmarked into the plain part.
|
|
86
|
+
result = extract_new_text(body, html_body=html_body)
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
`clean_for_scoring` takes the HTML signature container's visible text as an
|
|
90
|
+
optional hint, which removes signature furniture that carries no `-- `
|
|
91
|
+
delimiter and no recognizable contact shape:
|
|
92
|
+
|
|
93
|
+
```python
|
|
94
|
+
from email_reply_extractor import split_html_parts
|
|
95
|
+
|
|
96
|
+
parts = split_html_parts(html_body)
|
|
97
|
+
cleaned = clean_for_scoring(result.text, html_signature_text=parts.signature_text)
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
## Public API
|
|
101
|
+
|
|
102
|
+
| Name | What it is |
|
|
103
|
+
|---|---|
|
|
104
|
+
| `extract_new_text(body, parent_body=None, html_body=None)` | Stage 1; returns an `ExtractionResult`. |
|
|
105
|
+
| `ExtractionResult` | Frozen dataclass: `text`, `method`, `status`. |
|
|
106
|
+
| `STATUS_OK`, `STATUS_EMPTY`, `STATUS_FAILED` | The three `status` values. |
|
|
107
|
+
| `EXTRACTION_VERSION` | `int` generation stamp of the text-deriving routine. |
|
|
108
|
+
| `clean_for_scoring(extracted_text, html_signature_text=None)` | Stage 2; returns a `CleanResult`. |
|
|
109
|
+
| `CleanResult` | Frozen dataclass: `text`, `ignored_lines`. |
|
|
110
|
+
| `html_to_text(html)` | Visible text of an HTML part, block structure as newlines. |
|
|
111
|
+
| `split_html_parts(html)` | The same text split into `HtmlParts(novel_text, quoted_text, signature_text)`. |
|
|
112
|
+
| `HtmlParts` | Frozen dataclass returned by `split_html_parts`. |
|
|
113
|
+
| `strip_parent_content(text, parent_text)` | Remove lines of `text` provably drawn from `parent_text`. |
|
|
114
|
+
| `__version__` | The package version. |
|
|
115
|
+
|
|
116
|
+
### Status values
|
|
117
|
+
|
|
118
|
+
- `ok` — text was extracted.
|
|
119
|
+
- `empty` — nothing was left: a blank body, an HTML-only message with no usable
|
|
120
|
+
HTML text, or a body that reduced to quotes alone.
|
|
121
|
+
- `failed` — extraction raised; `text` is empty.
|
|
122
|
+
|
|
123
|
+
### Method strings
|
|
124
|
+
|
|
125
|
+
`ExtractionResult.method` records which path produced the text, so a result is
|
|
126
|
+
auditable after the fact. The base value is one of:
|
|
127
|
+
|
|
128
|
+
- `none` — no body to work from.
|
|
129
|
+
- `erp` — the vendored fragment scan needed no cleanup.
|
|
130
|
+
- `erp+custom` — the fragment scan plus the custom cleanup pass.
|
|
131
|
+
- `custom-fallback` — the over-strip guard fired, and the custom pass ran on
|
|
132
|
+
the whole body.
|
|
133
|
+
- `failed` — extraction raised.
|
|
134
|
+
|
|
135
|
+
Three modifiers combine with the base value:
|
|
136
|
+
|
|
137
|
+
- `html-` prefix — the text was derived from the HTML part, either because
|
|
138
|
+
there was no usable plain body or because a flattened plain body was
|
|
139
|
+
replaced (for example `html-erp+custom`).
|
|
140
|
+
- `+parent-diff` suffix — the parent-diff assist removed content (for example
|
|
141
|
+
`erp+parent-diff`).
|
|
142
|
+
- `+html-quote` suffix — the HTML quoted-text oracle removed content from a
|
|
143
|
+
plain extraction.
|
|
144
|
+
|
|
145
|
+
## Extraction version
|
|
146
|
+
|
|
147
|
+
`EXTRACTION_VERSION` is an `int` identifying the generation of the routine that
|
|
148
|
+
derives the text. It is independent of the package version, only ever
|
|
149
|
+
increases, and is incremented whenever a change to the extraction, cleaning,
|
|
150
|
+
HTML or vendored-scanner modules can alter the derived text. It is currently
|
|
151
|
+
**5**.
|
|
152
|
+
|
|
153
|
+
An application that stores derived text should store `EXTRACTION_VERSION`
|
|
154
|
+
alongside it and compare the stored stamp with the running constant using `<`
|
|
155
|
+
to decide whether the text needs re-deriving. The full contract, including the
|
|
156
|
+
digest that pins the behavior, is in [docs/versioning.md](docs/versioning.md).
|
|
157
|
+
Generation 5 (shipped in v1.1.0) and the work deferred from it are described in
|
|
158
|
+
[docs/generation-5.md](docs/generation-5.md). Because a generation bump can
|
|
159
|
+
move bytes without changing substance, the package exports `tolerant_lines`
|
|
160
|
+
and `texts_equivalent` so a consumer can re-derive stored text and keep
|
|
161
|
+
expensive downstream results (for example a paid classifier score) when only
|
|
162
|
+
whitespace moved.
|
|
163
|
+
|
|
164
|
+
## Supported Python versions
|
|
165
|
+
|
|
166
|
+
Python 3.10 through 3.14; each is exercised in CI.
|
|
167
|
+
|
|
168
|
+
## Comparison with alternatives
|
|
169
|
+
|
|
170
|
+
The two most widely used Python libraries for this task are
|
|
171
|
+
[email-reply-parser](https://github.com/zapier/email-reply-parser) (Zapier's
|
|
172
|
+
port of GitHub's Email Reply Parser) and
|
|
173
|
+
[talon](https://github.com/mailgun/talon) (Mailgun). The table states each
|
|
174
|
+
project's documented behavior; release data is from PyPI as of September 2026.
|
|
175
|
+
|
|
176
|
+
| | email-reply-extractor | email-reply-parser | talon |
|
|
177
|
+
|---|---|---|---|
|
|
178
|
+
| Extracts | The author's full novel content (stage 1), then the substantive prose alone (stage 2) | The latest reply, quoted text and signatures hidden | The reply with quotations removed; signatures separately |
|
|
179
|
+
| Signature handling | Kept in stage 1, removed and reported line-by-line in stage 2 | Hidden along with the reply's quoted fragments; not reported | Regex ("brute force") extractor, plus an SVM machine-learning extractor |
|
|
180
|
+
| HTML part | Used as a structural oracle: as the body when no usable plain part exists, replacing a flattened plain part, or removing a quoted message leaked unmarked into the plain part | Not handled | Quotation extraction has an HTML mode (lxml-based) |
|
|
181
|
+
| Thread parent | Optional parent-diff assist removes a quoted previous message that carries no `>` markers | Not used | Not used |
|
|
182
|
+
| Quote headers | Outlook-style blocks, including German and Chinese label sets, folded and double-spaced renderings | `From:`/`Sent:`/`To:`/`Subject:` fragment detection, English only | Delimiter patterns (e.g. `-----Original Message-----`), several languages |
|
|
183
|
+
| Output stability | Behavior pinned by an integer generation (`EXTRACTION_VERSION`) and a corpus digest test; whitespace-tolerant equivalence helpers for consumers | Not versioned separately | Not versioned separately |
|
|
184
|
+
| Test corpus | Hand-labeled real mailing-list messages (`tests/fixtures`) | Unit-test fixture emails | Unit-test fixture emails; ML training data |
|
|
185
|
+
| Runtime dependencies | None (standard library) | None (standard library) | lxml, regex, numpy, scipy, scikit-learn, chardet, and others |
|
|
186
|
+
| Latest release | 1.1.0 (2026) | 0.5.12 (2020) | 1.4.4 (2017) |
|
|
187
|
+
| License | MIT | MIT | Apache 2.0 |
|
|
188
|
+
|
|
189
|
+
The fragment scanner at this library's core derives from email-reply-parser —
|
|
190
|
+
see Provenance below — so on a plain-text body with English quoting the two
|
|
191
|
+
produce related results; the differences above are what this library adds on
|
|
192
|
+
top of that scan.
|
|
193
|
+
|
|
194
|
+
## Provenance
|
|
195
|
+
|
|
196
|
+
The code was extracted from
|
|
197
|
+
[mailing-list-ai-check](https://github.com/JayDaley/mailing-list-ai-check), an
|
|
198
|
+
application that checks mailing-list mail with an AI detector, where it shipped
|
|
199
|
+
as four extraction generations: 1 (initial release), 2 (from that
|
|
200
|
+
application's v1.2.0), 3 (from v1.11.0) and 4 (from v1.15.0). The hand-labeled
|
|
201
|
+
fixture corpus and its provenance came with it; see
|
|
202
|
+
[tests/fixtures/README.md](tests/fixtures/README.md).
|
|
203
|
+
|
|
204
|
+
The extraction also absorbed the application's only runtime dependency,
|
|
205
|
+
`email-reply-parser` 0.5.12 (MIT, Zapier's Python port of GitHub's Email Reply
|
|
206
|
+
Parser). Its fragment scanner is vendored, trimmed to the scan path this
|
|
207
|
+
package consumes, as `src/email_reply_extractor/_fragments.py`; two upstream
|
|
208
|
+
defects are preserved deliberately so the output does not move (see
|
|
209
|
+
[docs/generation-5.md](docs/generation-5.md)). The package therefore has no
|
|
210
|
+
runtime dependencies.
|
|
211
|
+
|
|
212
|
+
Two pieces of evidence support the claim that v1.0.0 derives the same text as
|
|
213
|
+
the application it came from: the recorded corpus digest carried over
|
|
214
|
+
unchanged, and a differential run over 110,725 real stored messages compared
|
|
215
|
+
the old (`email_reply_parser`) and new (vendored) implementations on stage-1
|
|
216
|
+
text, method and status and on the stage-2 composite, with zero mismatches.
|
|
217
|
+
|
|
218
|
+
## License
|
|
219
|
+
|
|
220
|
+
MIT. See [LICENSE](LICENSE).
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "email-reply-extractor"
|
|
3
|
+
dynamic = ["version"]
|
|
4
|
+
description = "Extract the author's newly written text from an email: quoted-text removal plus scoring-ready cleaning"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
authors = [{ name = "Jay Daley" }]
|
|
9
|
+
requires-python = ">=3.10"
|
|
10
|
+
dependencies = []
|
|
11
|
+
keywords = ["email", "reply", "quoting", "signature", "mailing-list", "text-extraction"]
|
|
12
|
+
classifiers = [
|
|
13
|
+
"Development Status :: 5 - Production/Stable",
|
|
14
|
+
"Intended Audience :: Developers",
|
|
15
|
+
"Operating System :: OS Independent",
|
|
16
|
+
"Programming Language :: Python :: 3",
|
|
17
|
+
"Topic :: Communications :: Email",
|
|
18
|
+
"Topic :: Text Processing",
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
[project.urls]
|
|
22
|
+
Homepage = "https://github.com/JayDaley/email-reply-extractor"
|
|
23
|
+
Changelog = "https://github.com/JayDaley/email-reply-extractor/blob/main/CHANGELOG.md"
|
|
24
|
+
|
|
25
|
+
[project.optional-dependencies]
|
|
26
|
+
dev = [
|
|
27
|
+
"pytest>=8.0",
|
|
28
|
+
"ruff>=0.6",
|
|
29
|
+
]
|
|
30
|
+
|
|
31
|
+
[build-system]
|
|
32
|
+
requires = ["setuptools>=77"]
|
|
33
|
+
build-backend = "setuptools.build_meta"
|
|
34
|
+
|
|
35
|
+
[tool.setuptools.dynamic]
|
|
36
|
+
version = {attr = "email_reply_extractor.__version__"}
|
|
37
|
+
|
|
38
|
+
[tool.setuptools.packages.find]
|
|
39
|
+
where = ["src"]
|
|
40
|
+
|
|
41
|
+
[tool.setuptools.package-data]
|
|
42
|
+
email_reply_extractor = ["py.typed"]
|
|
43
|
+
|
|
44
|
+
[tool.ruff]
|
|
45
|
+
line-length = 100
|
|
46
|
+
|
|
47
|
+
[tool.ruff.lint]
|
|
48
|
+
# The classic default rule set, pinned so a ruff upgrade cannot silently widen
|
|
49
|
+
# it; the vendored modules predate the newer defaults.
|
|
50
|
+
select = ["E4", "E7", "E9", "F"]
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""email-reply-extractor: recover the author's newly written text from an email.
|
|
2
|
+
|
|
3
|
+
A two-stage, pure-stdlib pipeline for mailing-list mail:
|
|
4
|
+
|
|
5
|
+
- **Stage 1** (:func:`extract_new_text`) removes quoted text, attribution
|
|
6
|
+
lines, forwarded/quote-header blocks and other reproduced content, keeping
|
|
7
|
+
the author's full novel content — greetings, sign-offs and signatures
|
|
8
|
+
included.
|
|
9
|
+
- **Stage 2** (:func:`clean_for_scoring`) strips that formulaic furniture from
|
|
10
|
+
the stage-1 text, leaving only the author's substantive prose, and reports
|
|
11
|
+
which lines it removed.
|
|
12
|
+
|
|
13
|
+
:data:`EXTRACTION_VERSION` identifies the generation of the routine that
|
|
14
|
+
derives the text; it is independent of the package version and only ever
|
|
15
|
+
increases. See ``CONTRIBUTING.md`` for the contract.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from .cleaning import CleanResult, clean_for_scoring
|
|
19
|
+
from .equivalence import texts_equivalent, tolerant_lines
|
|
20
|
+
from .extraction import (
|
|
21
|
+
EXTRACTION_VERSION,
|
|
22
|
+
STATUS_EMPTY,
|
|
23
|
+
STATUS_FAILED,
|
|
24
|
+
STATUS_OK,
|
|
25
|
+
ExtractionResult,
|
|
26
|
+
extract_new_text,
|
|
27
|
+
strip_parent_content,
|
|
28
|
+
)
|
|
29
|
+
from .html_text import HtmlParts, html_to_text, split_html_parts
|
|
30
|
+
|
|
31
|
+
__version__ = "1.1.0"
|
|
32
|
+
|
|
33
|
+
__all__ = [
|
|
34
|
+
"EXTRACTION_VERSION",
|
|
35
|
+
"STATUS_EMPTY",
|
|
36
|
+
"STATUS_FAILED",
|
|
37
|
+
"STATUS_OK",
|
|
38
|
+
"CleanResult",
|
|
39
|
+
"ExtractionResult",
|
|
40
|
+
"HtmlParts",
|
|
41
|
+
"__version__",
|
|
42
|
+
"clean_for_scoring",
|
|
43
|
+
"extract_new_text",
|
|
44
|
+
"html_to_text",
|
|
45
|
+
"split_html_parts",
|
|
46
|
+
"strip_parent_content",
|
|
47
|
+
"texts_equivalent",
|
|
48
|
+
"tolerant_lines",
|
|
49
|
+
]
|