email-reply-extractor 1.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (22) hide show
  1. email_reply_extractor-1.1.0/LICENSE +21 -0
  2. email_reply_extractor-1.1.0/PKG-INFO +243 -0
  3. email_reply_extractor-1.1.0/README.md +220 -0
  4. email_reply_extractor-1.1.0/pyproject.toml +50 -0
  5. email_reply_extractor-1.1.0/setup.cfg +4 -0
  6. email_reply_extractor-1.1.0/src/email_reply_extractor/__init__.py +49 -0
  7. email_reply_extractor-1.1.0/src/email_reply_extractor/_fragments.py +130 -0
  8. email_reply_extractor-1.1.0/src/email_reply_extractor/cleaning.py +601 -0
  9. email_reply_extractor-1.1.0/src/email_reply_extractor/equivalence.py +43 -0
  10. email_reply_extractor-1.1.0/src/email_reply_extractor/extraction.py +1007 -0
  11. email_reply_extractor-1.1.0/src/email_reply_extractor/html_text.py +427 -0
  12. email_reply_extractor-1.1.0/src/email_reply_extractor/py.typed +0 -0
  13. email_reply_extractor-1.1.0/src/email_reply_extractor.egg-info/PKG-INFO +243 -0
  14. email_reply_extractor-1.1.0/src/email_reply_extractor.egg-info/SOURCES.txt +20 -0
  15. email_reply_extractor-1.1.0/src/email_reply_extractor.egg-info/dependency_links.txt +1 -0
  16. email_reply_extractor-1.1.0/src/email_reply_extractor.egg-info/requires.txt +4 -0
  17. email_reply_extractor-1.1.0/src/email_reply_extractor.egg-info/top_level.txt +1 -0
  18. email_reply_extractor-1.1.0/tests/test_cleaning.py +490 -0
  19. email_reply_extractor-1.1.0/tests/test_equivalence.py +30 -0
  20. email_reply_extractor-1.1.0/tests/test_extraction.py +962 -0
  21. email_reply_extractor-1.1.0/tests/test_extraction_version.py +93 -0
  22. email_reply_extractor-1.1.0/tests/test_html_text.py +234 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Jay Daley
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,243 @@
1
+ Metadata-Version: 2.4
2
+ Name: email-reply-extractor
3
+ Version: 1.1.0
4
+ Summary: Extract the author's newly written text from an email: quoted-text removal plus scoring-ready cleaning
5
+ Author: Jay Daley
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/JayDaley/email-reply-extractor
8
+ Project-URL: Changelog, https://github.com/JayDaley/email-reply-extractor/blob/main/CHANGELOG.md
9
+ Keywords: email,reply,quoting,signature,mailing-list,text-extraction
10
+ Classifier: Development Status :: 5 - Production/Stable
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: Operating System :: OS Independent
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Topic :: Communications :: Email
15
+ Classifier: Topic :: Text Processing
16
+ Requires-Python: >=3.10
17
+ Description-Content-Type: text/markdown
18
+ License-File: LICENSE
19
+ Provides-Extra: dev
20
+ Requires-Dist: pytest>=8.0; extra == "dev"
21
+ Requires-Dist: ruff>=0.6; extra == "dev"
22
+ Dynamic: license-file
23
+
24
+ # email-reply-extractor
25
+
26
+ Recover the author's newly written text from an email — typically mailing-list
27
+ mail — in two stages. Pure standard library, no runtime dependencies,
28
+ Python 3.10 or later.
29
+
30
+ ## What it does
31
+
32
+ Stage 1, `extract_new_text`, removes everything the author did not write in
33
+ this message: quoted lines (any indentation, including the `...>` form that
34
+ survives `format=flowed` rewrapping), attribution lines ("On … wrote:", the
35
+ German "Am … schrieb", the Chinese Outlook and Alibaba Mail forms),
36
+ forwarded/quote-header blocks, "Original message" dividers, the quoted thread
37
+ that follows a sign-off boundary, and — when the caller supplies them —
38
+ content provably present in the thread parent or in the HTML part's
39
+ blockquotes. It **keeps** the author's greetings, sign-offs and signature
40
+ blocks.
41
+
42
+ Stage 2, `clean_for_scoring`, removes that formulaic furniture from stage-1
43
+ text: greetings, sign-offs, signature blocks, mailing-list footers, mobile
44
+ taglines and trailing legal disclaimers. It also reports which non-blank input
45
+ lines it removed.
46
+
47
+ The split is deliberate. The stage-1 text is the author's full novel content,
48
+ which is what an application should store and display; the stage-2 text is a
49
+ strict, documented subset of it, which is what an application should feed to a
50
+ downstream consumer such as a classifier. Because stage 2 reports the lines it
51
+ dropped, a stage-1 view can mark exactly what was set aside.
52
+
53
+ Both stages are pure and I/O-free. `extract_new_text` never raises: an
54
+ unexpected error is reported as `status="failed"` so a single bad message
55
+ cannot stall a pipeline.
56
+
57
+ The rules of both stages are documented in [docs/extraction.md](docs/extraction.md).
58
+
59
+ ## Install
60
+
61
+ Not yet published on PyPI. Install from the repository:
62
+
63
+ ```
64
+ pip install git+https://github.com/JayDaley/email-reply-extractor
65
+ ```
66
+
67
+ ## Quickstart
68
+
69
+ ```python
70
+ from email_reply_extractor import clean_for_scoring, extract_new_text
71
+
72
+ body = """Hi Dan,
73
+
74
+ Two comments below.
75
+
76
+ On Tue, Jul 7, 2026, at 03:25, Dan Wing wrote:
77
+ > I would lean heavily towards solution (2).
78
+
79
+ Agreed, and the draft already says so.
80
+
81
+ Best,
82
+ Lucas
83
+ """
84
+
85
+ result = extract_new_text(body)
86
+ result.status # 'ok'
87
+ result.method # 'erp'
88
+ result.text # greeting + both prose lines + the "Best, / Lucas" sign-off
89
+
90
+ cleaned = clean_for_scoring(result.text)
91
+ cleaned.text # 'Two comments below.\nAgreed, and the draft already says so.'
92
+ cleaned.ignored_lines # [0, 5, 6] — the greeting and the two sign-off lines
93
+ ```
94
+
95
+ The quote and its attribution line are gone from `result.text`; the greeting
96
+ and the sign-off are still there, and `clean_for_scoring` is what removes them.
97
+
98
+ Two optional arguments improve the result when the caller can supply them, and
99
+ change nothing when they are omitted:
100
+
101
+ ```python
102
+ # The thread parent's raw body, resolved by the caller from In-Reply-To.
103
+ # Removes a quoted previous message that carries no '>' markers at all.
104
+ result = extract_new_text(body, parent_body=parent_body)
105
+
106
+ # The message's decoded text/html part. Used as a structural oracle: as the
107
+ # body when there is no usable plain part, as a replacement for a flattened
108
+ # plain part, or to remove a quoted message leaked unmarked into the plain part.
109
+ result = extract_new_text(body, html_body=html_body)
110
+ ```
111
+
112
+ `clean_for_scoring` takes the HTML signature container's visible text as an
113
+ optional hint, which removes signature furniture that carries no `-- `
114
+ delimiter and no recognizable contact shape:
115
+
116
+ ```python
117
+ from email_reply_extractor import split_html_parts
118
+
119
+ parts = split_html_parts(html_body)
120
+ cleaned = clean_for_scoring(result.text, html_signature_text=parts.signature_text)
121
+ ```
122
+
123
+ ## Public API
124
+
125
+ | Name | What it is |
126
+ |---|---|
127
+ | `extract_new_text(body, parent_body=None, html_body=None)` | Stage 1; returns an `ExtractionResult`. |
128
+ | `ExtractionResult` | Frozen dataclass: `text`, `method`, `status`. |
129
+ | `STATUS_OK`, `STATUS_EMPTY`, `STATUS_FAILED` | The three `status` values. |
130
+ | `EXTRACTION_VERSION` | `int` generation stamp of the text-deriving routine. |
131
+ | `clean_for_scoring(extracted_text, html_signature_text=None)` | Stage 2; returns a `CleanResult`. |
132
+ | `CleanResult` | Frozen dataclass: `text`, `ignored_lines`. |
133
+ | `html_to_text(html)` | Visible text of an HTML part, block structure as newlines. |
134
+ | `split_html_parts(html)` | The same text split into `HtmlParts(novel_text, quoted_text, signature_text)`. |
135
+ | `HtmlParts` | Frozen dataclass returned by `split_html_parts`. |
136
+ | `strip_parent_content(text, parent_text)` | Remove lines of `text` provably drawn from `parent_text`. |
137
+ | `__version__` | The package version. |
138
+
139
+ ### Status values
140
+
141
+ - `ok` — text was extracted.
142
+ - `empty` — nothing was left: a blank body, an HTML-only message with no usable
143
+ HTML text, or a body that reduced to quotes alone.
144
+ - `failed` — extraction raised; `text` is empty.
145
+
146
+ ### Method strings
147
+
148
+ `ExtractionResult.method` records which path produced the text, so a result is
149
+ auditable after the fact. The base value is one of:
150
+
151
+ - `none` — no body to work from.
152
+ - `erp` — the vendored fragment scan needed no cleanup.
153
+ - `erp+custom` — the fragment scan plus the custom cleanup pass.
154
+ - `custom-fallback` — the over-strip guard fired, and the custom pass ran on
155
+ the whole body.
156
+ - `failed` — extraction raised.
157
+
158
+ Three modifiers combine with the base value:
159
+
160
+ - `html-` prefix — the text was derived from the HTML part, either because
161
+ there was no usable plain body or because a flattened plain body was
162
+ replaced (for example `html-erp+custom`).
163
+ - `+parent-diff` suffix — the parent-diff assist removed content (for example
164
+ `erp+parent-diff`).
165
+ - `+html-quote` suffix — the HTML quoted-text oracle removed content from a
166
+ plain extraction.
167
+
168
+ ## Extraction version
169
+
170
+ `EXTRACTION_VERSION` is an `int` identifying the generation of the routine that
171
+ derives the text. It is independent of the package version, only ever
172
+ increases, and is incremented whenever a change to the extraction, cleaning,
173
+ HTML or vendored-scanner modules can alter the derived text. It is currently
174
+ **5**.
175
+
176
+ An application that stores derived text should store `EXTRACTION_VERSION`
177
+ alongside it and compare the stored stamp with the running constant using `<`
178
+ to decide whether the text needs re-deriving. The full contract, including the
179
+ digest that pins the behavior, is in [docs/versioning.md](docs/versioning.md).
180
+ Generation 5 (shipped in v1.1.0) and the work deferred from it are described in
181
+ [docs/generation-5.md](docs/generation-5.md). Because a generation bump can
182
+ move bytes without changing substance, the package exports `tolerant_lines`
183
+ and `texts_equivalent` so a consumer can re-derive stored text and keep
184
+ expensive downstream results (for example a paid classifier score) when only
185
+ whitespace moved.
186
+
187
+ ## Supported Python versions
188
+
189
+ Python 3.10 through 3.14; each is exercised in CI.
190
+
191
+ ## Comparison with alternatives
192
+
193
+ The two most widely used Python libraries for this task are
194
+ [email-reply-parser](https://github.com/zapier/email-reply-parser) (Zapier's
195
+ port of GitHub's Email Reply Parser) and
196
+ [talon](https://github.com/mailgun/talon) (Mailgun). The table states each
197
+ project's documented behavior; release data is from PyPI as of September 2026.
198
+
199
+ | | email-reply-extractor | email-reply-parser | talon |
200
+ |---|---|---|---|
201
+ | Extracts | The author's full novel content (stage 1), then the substantive prose alone (stage 2) | The latest reply, quoted text and signatures hidden | The reply with quotations removed; signatures separately |
202
+ | Signature handling | Kept in stage 1, removed and reported line-by-line in stage 2 | Hidden along with the reply's quoted fragments; not reported | Regex ("brute force") extractor, plus an SVM machine-learning extractor |
203
+ | HTML part | Used as a structural oracle: as the body when no usable plain part exists, replacing a flattened plain part, or removing a quoted message leaked unmarked into the plain part | Not handled | Quotation extraction has an HTML mode (lxml-based) |
204
+ | Thread parent | Optional parent-diff assist removes a quoted previous message that carries no `>` markers | Not used | Not used |
205
+ | Quote headers | Outlook-style blocks, including German and Chinese label sets, folded and double-spaced renderings | `From:`/`Sent:`/`To:`/`Subject:` fragment detection, English only | Delimiter patterns (e.g. `-----Original Message-----`), several languages |
206
+ | Output stability | Behavior pinned by an integer generation (`EXTRACTION_VERSION`) and a corpus digest test; whitespace-tolerant equivalence helpers for consumers | Not versioned separately | Not versioned separately |
207
+ | Test corpus | Hand-labeled real mailing-list messages (`tests/fixtures`) | Unit-test fixture emails | Unit-test fixture emails; ML training data |
208
+ | Runtime dependencies | None (standard library) | None (standard library) | lxml, regex, numpy, scipy, scikit-learn, chardet, and others |
209
+ | Latest release | 1.1.0 (2026) | 0.5.12 (2020) | 1.4.4 (2017) |
210
+ | License | MIT | MIT | Apache 2.0 |
211
+
212
+ The fragment scanner at this library's core derives from email-reply-parser —
213
+ see Provenance below — so on a plain-text body with English quoting the two
214
+ produce related results; the differences above are what this library adds on
215
+ top of that scan.
216
+
217
+ ## Provenance
218
+
219
+ The code was extracted from
220
+ [mailing-list-ai-check](https://github.com/JayDaley/mailing-list-ai-check), an
221
+ application that checks mailing-list mail with an AI detector, where it shipped
222
+ as four extraction generations: 1 (initial release), 2 (from that
223
+ application's v1.2.0), 3 (from v1.11.0) and 4 (from v1.15.0). The hand-labeled
224
+ fixture corpus and its provenance came with it; see
225
+ [tests/fixtures/README.md](tests/fixtures/README.md).
226
+
227
+ The extraction also absorbed the application's only runtime dependency,
228
+ `email-reply-parser` 0.5.12 (MIT, Zapier's Python port of GitHub's Email Reply
229
+ Parser). Its fragment scanner is vendored, trimmed to the scan path this
230
+ package consumes, as `src/email_reply_extractor/_fragments.py`; two upstream
231
+ defects are preserved deliberately so the output does not move (see
232
+ [docs/generation-5.md](docs/generation-5.md)). The package therefore has no
233
+ runtime dependencies.
234
+
235
+ Two pieces of evidence support the claim that v1.0.0 derives the same text as
236
+ the application it came from: the recorded corpus digest carried over
237
+ unchanged, and a differential run over 110,725 real stored messages compared
238
+ the old (`email_reply_parser`) and new (vendored) implementations on stage-1
239
+ text, method and status and on the stage-2 composite, with zero mismatches.
240
+
241
+ ## License
242
+
243
+ MIT. See [LICENSE](LICENSE).
@@ -0,0 +1,220 @@
1
+ # email-reply-extractor
2
+
3
+ Recover the author's newly written text from an email — typically mailing-list
4
+ mail — in two stages. Pure standard library, no runtime dependencies,
5
+ Python 3.10 or later.
6
+
7
+ ## What it does
8
+
9
+ Stage 1, `extract_new_text`, removes everything the author did not write in
10
+ this message: quoted lines (any indentation, including the `...>` form that
11
+ survives `format=flowed` rewrapping), attribution lines ("On … wrote:", the
12
+ German "Am … schrieb", the Chinese Outlook and Alibaba Mail forms),
13
+ forwarded/quote-header blocks, "Original message" dividers, the quoted thread
14
+ that follows a sign-off boundary, and — when the caller supplies them —
15
+ content provably present in the thread parent or in the HTML part's
16
+ blockquotes. It **keeps** the author's greetings, sign-offs and signature
17
+ blocks.
18
+
19
+ Stage 2, `clean_for_scoring`, removes that formulaic furniture from stage-1
20
+ text: greetings, sign-offs, signature blocks, mailing-list footers, mobile
21
+ taglines and trailing legal disclaimers. It also reports which non-blank input
22
+ lines it removed.
23
+
24
+ The split is deliberate. The stage-1 text is the author's full novel content,
25
+ which is what an application should store and display; the stage-2 text is a
26
+ strict, documented subset of it, which is what an application should feed to a
27
+ downstream consumer such as a classifier. Because stage 2 reports the lines it
28
+ dropped, a stage-1 view can mark exactly what was set aside.
29
+
30
+ Both stages are pure and I/O-free. `extract_new_text` never raises: an
31
+ unexpected error is reported as `status="failed"` so a single bad message
32
+ cannot stall a pipeline.
33
+
34
+ The rules of both stages are documented in [docs/extraction.md](docs/extraction.md).
35
+
36
+ ## Install
37
+
38
+ Not yet published on PyPI. Install from the repository:
39
+
40
+ ```
41
+ pip install git+https://github.com/JayDaley/email-reply-extractor
42
+ ```
43
+
44
+ ## Quickstart
45
+
46
+ ```python
47
+ from email_reply_extractor import clean_for_scoring, extract_new_text
48
+
49
+ body = """Hi Dan,
50
+
51
+ Two comments below.
52
+
53
+ On Tue, Jul 7, 2026, at 03:25, Dan Wing wrote:
54
+ > I would lean heavily towards solution (2).
55
+
56
+ Agreed, and the draft already says so.
57
+
58
+ Best,
59
+ Lucas
60
+ """
61
+
62
+ result = extract_new_text(body)
63
+ result.status # 'ok'
64
+ result.method # 'erp'
65
+ result.text # greeting + both prose lines + the "Best, / Lucas" sign-off
66
+
67
+ cleaned = clean_for_scoring(result.text)
68
+ cleaned.text # 'Two comments below.\nAgreed, and the draft already says so.'
69
+ cleaned.ignored_lines # [0, 5, 6] — the greeting and the two sign-off lines
70
+ ```
71
+
72
+ The quote and its attribution line are gone from `result.text`; the greeting
73
+ and the sign-off are still there, and `clean_for_scoring` is what removes them.
74
+
75
+ Two optional arguments improve the result when the caller can supply them, and
76
+ change nothing when they are omitted:
77
+
78
+ ```python
79
+ # The thread parent's raw body, resolved by the caller from In-Reply-To.
80
+ # Removes a quoted previous message that carries no '>' markers at all.
81
+ result = extract_new_text(body, parent_body=parent_body)
82
+
83
+ # The message's decoded text/html part. Used as a structural oracle: as the
84
+ # body when there is no usable plain part, as a replacement for a flattened
85
+ # plain part, or to remove a quoted message leaked unmarked into the plain part.
86
+ result = extract_new_text(body, html_body=html_body)
87
+ ```
88
+
89
+ `clean_for_scoring` takes the HTML signature container's visible text as an
90
+ optional hint, which removes signature furniture that carries no `-- `
91
+ delimiter and no recognizable contact shape:
92
+
93
+ ```python
94
+ from email_reply_extractor import split_html_parts
95
+
96
+ parts = split_html_parts(html_body)
97
+ cleaned = clean_for_scoring(result.text, html_signature_text=parts.signature_text)
98
+ ```
99
+
100
+ ## Public API
101
+
102
+ | Name | What it is |
103
+ |---|---|
104
+ | `extract_new_text(body, parent_body=None, html_body=None)` | Stage 1; returns an `ExtractionResult`. |
105
+ | `ExtractionResult` | Frozen dataclass: `text`, `method`, `status`. |
106
+ | `STATUS_OK`, `STATUS_EMPTY`, `STATUS_FAILED` | The three `status` values. |
107
+ | `EXTRACTION_VERSION` | `int` generation stamp of the text-deriving routine. |
108
+ | `clean_for_scoring(extracted_text, html_signature_text=None)` | Stage 2; returns a `CleanResult`. |
109
+ | `CleanResult` | Frozen dataclass: `text`, `ignored_lines`. |
110
+ | `html_to_text(html)` | Visible text of an HTML part, block structure as newlines. |
111
+ | `split_html_parts(html)` | The same text split into `HtmlParts(novel_text, quoted_text, signature_text)`. |
112
+ | `HtmlParts` | Frozen dataclass returned by `split_html_parts`. |
113
+ | `strip_parent_content(text, parent_text)` | Remove lines of `text` provably drawn from `parent_text`. |
114
+ | `__version__` | The package version. |
115
+
116
+ ### Status values
117
+
118
+ - `ok` — text was extracted.
119
+ - `empty` — nothing was left: a blank body, an HTML-only message with no usable
120
+ HTML text, or a body that reduced to quotes alone.
121
+ - `failed` — extraction raised; `text` is empty.
122
+
123
+ ### Method strings
124
+
125
+ `ExtractionResult.method` records which path produced the text, so a result is
126
+ auditable after the fact. The base value is one of:
127
+
128
+ - `none` — no body to work from.
129
+ - `erp` — the vendored fragment scan needed no cleanup.
130
+ - `erp+custom` — the fragment scan plus the custom cleanup pass.
131
+ - `custom-fallback` — the over-strip guard fired, and the custom pass ran on
132
+ the whole body.
133
+ - `failed` — extraction raised.
134
+
135
+ Three modifiers combine with the base value:
136
+
137
+ - `html-` prefix — the text was derived from the HTML part, either because
138
+ there was no usable plain body or because a flattened plain body was
139
+ replaced (for example `html-erp+custom`).
140
+ - `+parent-diff` suffix — the parent-diff assist removed content (for example
141
+ `erp+parent-diff`).
142
+ - `+html-quote` suffix — the HTML quoted-text oracle removed content from a
143
+ plain extraction.
144
+
145
+ ## Extraction version
146
+
147
+ `EXTRACTION_VERSION` is an `int` identifying the generation of the routine that
148
+ derives the text. It is independent of the package version, only ever
149
+ increases, and is incremented whenever a change to the extraction, cleaning,
150
+ HTML or vendored-scanner modules can alter the derived text. It is currently
151
+ **5**.
152
+
153
+ An application that stores derived text should store `EXTRACTION_VERSION`
154
+ alongside it and compare the stored stamp with the running constant using `<`
155
+ to decide whether the text needs re-deriving. The full contract, including the
156
+ digest that pins the behavior, is in [docs/versioning.md](docs/versioning.md).
157
+ Generation 5 (shipped in v1.1.0) and the work deferred from it are described in
158
+ [docs/generation-5.md](docs/generation-5.md). Because a generation bump can
159
+ move bytes without changing substance, the package exports `tolerant_lines`
160
+ and `texts_equivalent` so a consumer can re-derive stored text and keep
161
+ expensive downstream results (for example a paid classifier score) when only
162
+ whitespace moved.
163
+
164
+ ## Supported Python versions
165
+
166
+ Python 3.10 through 3.14; each is exercised in CI.
167
+
168
+ ## Comparison with alternatives
169
+
170
+ The two most widely used Python libraries for this task are
171
+ [email-reply-parser](https://github.com/zapier/email-reply-parser) (Zapier's
172
+ port of GitHub's Email Reply Parser) and
173
+ [talon](https://github.com/mailgun/talon) (Mailgun). The table states each
174
+ project's documented behavior; release data is from PyPI as of September 2026.
175
+
176
+ | | email-reply-extractor | email-reply-parser | talon |
177
+ |---|---|---|---|
178
+ | Extracts | The author's full novel content (stage 1), then the substantive prose alone (stage 2) | The latest reply, quoted text and signatures hidden | The reply with quotations removed; signatures separately |
179
+ | Signature handling | Kept in stage 1, removed and reported line-by-line in stage 2 | Hidden along with the reply's quoted fragments; not reported | Regex ("brute force") extractor, plus an SVM machine-learning extractor |
180
+ | HTML part | Used as a structural oracle: as the body when no usable plain part exists, replacing a flattened plain part, or removing a quoted message leaked unmarked into the plain part | Not handled | Quotation extraction has an HTML mode (lxml-based) |
181
+ | Thread parent | Optional parent-diff assist removes a quoted previous message that carries no `>` markers | Not used | Not used |
182
+ | Quote headers | Outlook-style blocks, including German and Chinese label sets, folded and double-spaced renderings | `From:`/`Sent:`/`To:`/`Subject:` fragment detection, English only | Delimiter patterns (e.g. `-----Original Message-----`), several languages |
183
+ | Output stability | Behavior pinned by an integer generation (`EXTRACTION_VERSION`) and a corpus digest test; whitespace-tolerant equivalence helpers for consumers | Not versioned separately | Not versioned separately |
184
+ | Test corpus | Hand-labeled real mailing-list messages (`tests/fixtures`) | Unit-test fixture emails | Unit-test fixture emails; ML training data |
185
+ | Runtime dependencies | None (standard library) | None (standard library) | lxml, regex, numpy, scipy, scikit-learn, chardet, and others |
186
+ | Latest release | 1.1.0 (2026) | 0.5.12 (2020) | 1.4.4 (2017) |
187
+ | License | MIT | MIT | Apache 2.0 |
188
+
189
+ The fragment scanner at this library's core derives from email-reply-parser —
190
+ see Provenance below — so on a plain-text body with English quoting the two
191
+ produce related results; the differences above are what this library adds on
192
+ top of that scan.
193
+
194
+ ## Provenance
195
+
196
+ The code was extracted from
197
+ [mailing-list-ai-check](https://github.com/JayDaley/mailing-list-ai-check), an
198
+ application that checks mailing-list mail with an AI detector, where it shipped
199
+ as four extraction generations: 1 (initial release), 2 (from that
200
+ application's v1.2.0), 3 (from v1.11.0) and 4 (from v1.15.0). The hand-labeled
201
+ fixture corpus and its provenance came with it; see
202
+ [tests/fixtures/README.md](tests/fixtures/README.md).
203
+
204
+ The extraction also absorbed the application's only runtime dependency,
205
+ `email-reply-parser` 0.5.12 (MIT, Zapier's Python port of GitHub's Email Reply
206
+ Parser). Its fragment scanner is vendored, trimmed to the scan path this
207
+ package consumes, as `src/email_reply_extractor/_fragments.py`; two upstream
208
+ defects are preserved deliberately so the output does not move (see
209
+ [docs/generation-5.md](docs/generation-5.md)). The package therefore has no
210
+ runtime dependencies.
211
+
212
+ Two pieces of evidence support the claim that v1.0.0 derives the same text as
213
+ the application it came from: the recorded corpus digest carried over
214
+ unchanged, and a differential run over 110,725 real stored messages compared
215
+ the old (`email_reply_parser`) and new (vendored) implementations on stage-1
216
+ text, method and status and on the stage-2 composite, with zero mismatches.
217
+
218
+ ## License
219
+
220
+ MIT. See [LICENSE](LICENSE).
@@ -0,0 +1,50 @@
1
+ [project]
2
+ name = "email-reply-extractor"
3
+ dynamic = ["version"]
4
+ description = "Extract the author's newly written text from an email: quoted-text removal plus scoring-ready cleaning"
5
+ readme = "README.md"
6
+ license = "MIT"
7
+ license-files = ["LICENSE"]
8
+ authors = [{ name = "Jay Daley" }]
9
+ requires-python = ">=3.10"
10
+ dependencies = []
11
+ keywords = ["email", "reply", "quoting", "signature", "mailing-list", "text-extraction"]
12
+ classifiers = [
13
+ "Development Status :: 5 - Production/Stable",
14
+ "Intended Audience :: Developers",
15
+ "Operating System :: OS Independent",
16
+ "Programming Language :: Python :: 3",
17
+ "Topic :: Communications :: Email",
18
+ "Topic :: Text Processing",
19
+ ]
20
+
21
+ [project.urls]
22
+ Homepage = "https://github.com/JayDaley/email-reply-extractor"
23
+ Changelog = "https://github.com/JayDaley/email-reply-extractor/blob/main/CHANGELOG.md"
24
+
25
+ [project.optional-dependencies]
26
+ dev = [
27
+ "pytest>=8.0",
28
+ "ruff>=0.6",
29
+ ]
30
+
31
+ [build-system]
32
+ requires = ["setuptools>=77"]
33
+ build-backend = "setuptools.build_meta"
34
+
35
+ [tool.setuptools.dynamic]
36
+ version = {attr = "email_reply_extractor.__version__"}
37
+
38
+ [tool.setuptools.packages.find]
39
+ where = ["src"]
40
+
41
+ [tool.setuptools.package-data]
42
+ email_reply_extractor = ["py.typed"]
43
+
44
+ [tool.ruff]
45
+ line-length = 100
46
+
47
+ [tool.ruff.lint]
48
+ # The classic default rule set, pinned so a ruff upgrade cannot silently widen
49
+ # it; the vendored modules predate the newer defaults.
50
+ select = ["E4", "E7", "E9", "F"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,49 @@
1
+ """email-reply-extractor: recover the author's newly written text from an email.
2
+
3
+ A two-stage, pure-stdlib pipeline for mailing-list mail:
4
+
5
+ - **Stage 1** (:func:`extract_new_text`) removes quoted text, attribution
6
+ lines, forwarded/quote-header blocks and other reproduced content, keeping
7
+ the author's full novel content — greetings, sign-offs and signatures
8
+ included.
9
+ - **Stage 2** (:func:`clean_for_scoring`) strips that formulaic furniture from
10
+ the stage-1 text, leaving only the author's substantive prose, and reports
11
+ which lines it removed.
12
+
13
+ :data:`EXTRACTION_VERSION` identifies the generation of the routine that
14
+ derives the text; it is independent of the package version and only ever
15
+ increases. See ``CONTRIBUTING.md`` for the contract.
16
+ """
17
+
18
+ from .cleaning import CleanResult, clean_for_scoring
19
+ from .equivalence import texts_equivalent, tolerant_lines
20
+ from .extraction import (
21
+ EXTRACTION_VERSION,
22
+ STATUS_EMPTY,
23
+ STATUS_FAILED,
24
+ STATUS_OK,
25
+ ExtractionResult,
26
+ extract_new_text,
27
+ strip_parent_content,
28
+ )
29
+ from .html_text import HtmlParts, html_to_text, split_html_parts
30
+
31
+ __version__ = "1.1.0"
32
+
33
+ __all__ = [
34
+ "EXTRACTION_VERSION",
35
+ "STATUS_EMPTY",
36
+ "STATUS_FAILED",
37
+ "STATUS_OK",
38
+ "CleanResult",
39
+ "ExtractionResult",
40
+ "HtmlParts",
41
+ "__version__",
42
+ "clean_for_scoring",
43
+ "extract_new_text",
44
+ "html_to_text",
45
+ "split_html_parts",
46
+ "strip_parent_content",
47
+ "texts_equivalent",
48
+ "tolerant_lines",
49
+ ]