epub2text 0.2.2__tar.gz → 0.2.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- epub2text-0.2.3/.codecrate.toml +10 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/.gitignore +1 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/PKG-INFO +2 -3
- {epub2text-0.2.2 → epub2text-0.2.3}/docs/installation.rst +0 -4
- {epub2text-0.2.2 → epub2text-0.2.3}/epub2text/_version.py +3 -3
- {epub2text-0.2.2 → epub2text-0.2.3}/epub2text/cleaner.py +0 -2
- {epub2text-0.2.2 → epub2text-0.2.3}/epub2text/formatters.py +17 -101
- {epub2text-0.2.2 → epub2text-0.2.3}/epub2text/parser.py +54 -116
- {epub2text-0.2.2 → epub2text-0.2.3}/epub2text.egg-info/PKG-INFO +2 -3
- {epub2text-0.2.2 → epub2text-0.2.3}/epub2text.egg-info/SOURCES.txt +1 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/epub2text.egg-info/requires.txt +1 -3
- {epub2text-0.2.2 → epub2text-0.2.3}/examples/demo.py +0 -16
- {epub2text-0.2.2 → epub2text-0.2.3}/pyproject.toml +1 -3
- {epub2text-0.2.2 → epub2text-0.2.3}/tests/test_epub_download.py +7 -2
- {epub2text-0.2.2 → epub2text-0.2.3}/tests/test_epub_generation.py +5 -12
- {epub2text-0.2.2 → epub2text-0.2.3}/tests/test_title_deduplication.py +3 -1
- {epub2text-0.2.2 → epub2text-0.2.3}/.coveragerc +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/.github/pytest.ini +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/.github/workflows/codecov.yml +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/.github/workflows/pre-commit.yml +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/.github/workflows/python-publish.yml +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/.github/workflows/tests.yml +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/.pre-commit-config.yaml +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/.prettierrc.yml +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/.readthedocs.yaml +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/.ruff.toml +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/LICENSE +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/README.md +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/docs/api.rst +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/docs/changelog.rst +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/docs/conf.py +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/docs/index.rst +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/docs/make.bat +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/docs/make.py +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/docs/requirements.txt +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/docs/usage.rst +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/epub2text/__init__.py +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/epub2text/__main__.py +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/epub2text/bookmarks.py +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/epub2text/cli.py +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/epub2text/models.py +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/epub2text/py.typed +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/epub2text/reader.py +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/epub2text.egg-info/dependency_links.txt +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/epub2text.egg-info/entry_points.txt +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/epub2text.egg-info/top_level.txt +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/examples/__init__.py +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/examples/__main__.py +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/requirements-test.txt +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/setup.cfg +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/setup.py +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/tests/__init__.py +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/tests/test_cleaner.py +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/tests/test_cli_ranges.py +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/tests/test_page_extraction_improvements.py +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/tests/test_parser_page_list.py +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/tests/test_parser_spine_slice.py +0 -0
- {epub2text-0.2.2 → epub2text-0.2.3}/tests/test_reader_bookmarks.py +0 -0
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
[codecrate]
|
|
2
|
+
output = "context_epub2text.md"
|
|
3
|
+
keep_docstrings = true
|
|
4
|
+
dedupe = true
|
|
5
|
+
metadata = false
|
|
6
|
+
respect_gitignore = true
|
|
7
|
+
exclude = ["examples", "*/.venv/*"]
|
|
8
|
+
include = ["**/*.py", "**/*.toml", "**/*.rst", "**/*.md"]
|
|
9
|
+
split_max_chars = 0
|
|
10
|
+
layout = "full"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: epub2text
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.3
|
|
4
4
|
Summary: A simple tool to extract text from EPUB files.
|
|
5
5
|
Author-email: Holger Nahrstaedt <nahrstaedt@gmail.com>
|
|
6
6
|
License: MIT License
|
|
@@ -45,10 +45,9 @@ Requires-Dist: rich>=13.0.0
|
|
|
45
45
|
Requires-Dist: ebooklib>=0.18
|
|
46
46
|
Requires-Dist: beautifulsoup4>=4.12.0
|
|
47
47
|
Requires-Dist: defusedxml>=0.7.1
|
|
48
|
+
Requires-Dist: phrasplit>=0.3.2
|
|
48
49
|
Provides-Extra: lxml
|
|
49
50
|
Requires-Dist: lxml>=4.9.0; extra == "lxml"
|
|
50
|
-
Provides-Extra: sentences
|
|
51
|
-
Requires-Dist: phrasplit>=0.1.0; extra == "sentences"
|
|
52
51
|
Provides-Extra: dev
|
|
53
52
|
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
54
53
|
Requires-Dist: ruff>=0.1.0; extra == "dev"
|
|
@@ -31,10 +31,6 @@ For better HTML parsing performance::
|
|
|
31
31
|
|
|
32
32
|
pip install epub2text[lxml]
|
|
33
33
|
|
|
34
|
-
For sentence-level formatting (requires phrasplit/spaCy)::
|
|
35
|
-
|
|
36
|
-
pip install epub2text[sentences]
|
|
37
|
-
|
|
38
34
|
After installing, download a spaCy language model::
|
|
39
35
|
|
|
40
36
|
python -m spacy download en_core_web_sm
|
|
@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
|
|
|
28
28
|
commit_id: COMMIT_ID
|
|
29
29
|
__commit_id__: COMMIT_ID
|
|
30
30
|
|
|
31
|
-
__version__ = version = '0.2.
|
|
32
|
-
__version_tuple__ = version_tuple = (0, 2,
|
|
31
|
+
__version__ = version = '0.2.3'
|
|
32
|
+
__version_tuple__ = version_tuple = (0, 2, 3)
|
|
33
33
|
|
|
34
|
-
__commit_id__ = commit_id = '
|
|
34
|
+
__commit_id__ = commit_id = 'g82e1d316b'
|
|
@@ -147,8 +147,6 @@ class TextCleaner:
|
|
|
147
147
|
def apply_gutenberg_spacing(self, text: str) -> str:
|
|
148
148
|
"""Apply Gutenberg Project formatting: two spaces after sentences and colons.
|
|
149
149
|
|
|
150
|
-
Uses phrasplit for accurate sentence boundary detection.
|
|
151
|
-
|
|
152
150
|
Args:
|
|
153
151
|
text: Text to format
|
|
154
152
|
|
|
@@ -1,70 +1,15 @@
|
|
|
1
1
|
"""Text formatting utilities for different output styles."""
|
|
2
2
|
|
|
3
3
|
import re
|
|
4
|
-
from typing import Callable, Optional
|
|
5
|
-
|
|
6
|
-
# Type aliases for phrasplit functions
|
|
7
|
-
_SplitFunc = Callable[[str, str], list[str]]
|
|
8
|
-
_SplitLongLinesFunc = Callable[[str, int, str], list[str]]
|
|
9
|
-
|
|
10
|
-
# Try to import phrasplit, fall back to None when not available
|
|
11
|
-
PHRASPLIT_AVAILABLE: bool
|
|
12
|
-
phrasplit_clauses: Optional[_SplitFunc]
|
|
13
|
-
phrasplit_long_lines: Optional[_SplitLongLinesFunc]
|
|
14
|
-
phrasplit_paragraphs: Optional[Callable[[str], list[str]]]
|
|
15
|
-
phrasplit_sentences: Optional[_SplitFunc]
|
|
16
|
-
|
|
17
|
-
try:
|
|
18
|
-
from phrasplit import ( # type: ignore[import-not-found]
|
|
19
|
-
split_clauses as _phrasplit_clauses,
|
|
20
|
-
)
|
|
21
|
-
from phrasplit import (
|
|
22
|
-
split_long_lines as _phrasplit_long_lines,
|
|
23
|
-
)
|
|
24
|
-
from phrasplit import (
|
|
25
|
-
split_paragraphs as _phrasplit_paragraphs,
|
|
26
|
-
)
|
|
27
|
-
from phrasplit import (
|
|
28
|
-
split_sentences as _phrasplit_sentences,
|
|
29
|
-
)
|
|
30
|
-
|
|
31
|
-
PHRASPLIT_AVAILABLE = True
|
|
32
|
-
phrasplit_clauses = _phrasplit_clauses
|
|
33
|
-
phrasplit_long_lines = _phrasplit_long_lines
|
|
34
|
-
phrasplit_paragraphs = _phrasplit_paragraphs
|
|
35
|
-
phrasplit_sentences = _phrasplit_sentences
|
|
36
|
-
except ImportError:
|
|
37
|
-
PHRASPLIT_AVAILABLE = False
|
|
38
|
-
phrasplit_clauses = None
|
|
39
|
-
phrasplit_long_lines = None
|
|
40
|
-
phrasplit_paragraphs = None
|
|
41
|
-
phrasplit_sentences = None
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
def _check_phrasplit() -> None:
|
|
45
|
-
"""Check if phrasplit is available, raise ImportError if not."""
|
|
46
|
-
if not PHRASPLIT_AVAILABLE:
|
|
47
|
-
raise ImportError(
|
|
48
|
-
"phrasplit is required for this feature. "
|
|
49
|
-
"Install with: pip install epub2text[sentences]"
|
|
50
|
-
)
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
def split_paragraphs(text: str) -> list[str]:
|
|
54
|
-
"""
|
|
55
|
-
Split text into paragraphs (separated by double newlines).
|
|
56
4
|
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
# Simple implementation that doesn't require phrasplit
|
|
66
|
-
paragraphs = re.split(r"\n\s*\n", text)
|
|
67
|
-
return [p.strip() for p in paragraphs if p.strip()]
|
|
5
|
+
from phrasplit import (
|
|
6
|
+
split_clauses,
|
|
7
|
+
split_paragraphs,
|
|
8
|
+
split_sentences,
|
|
9
|
+
)
|
|
10
|
+
from phrasplit import (
|
|
11
|
+
split_long_lines as _phrasplit_split_long_lines,
|
|
12
|
+
)
|
|
68
13
|
|
|
69
14
|
|
|
70
15
|
def collapse_paragraph(paragraph: str) -> str:
|
|
@@ -146,8 +91,6 @@ def format_sentences(
|
|
|
146
91
|
"""
|
|
147
92
|
Format text with one sentence per line.
|
|
148
93
|
|
|
149
|
-
Requires phrasplit to be installed.
|
|
150
|
-
|
|
151
94
|
Args:
|
|
152
95
|
text: Input text with paragraph breaks
|
|
153
96
|
separator: String to prepend at paragraph boundaries (default: " ")
|
|
@@ -157,9 +100,6 @@ def format_sentences(
|
|
|
157
100
|
Text with one sentence per line, separator at paragraph boundaries.
|
|
158
101
|
Chapter titles are preserved.
|
|
159
102
|
"""
|
|
160
|
-
_check_phrasplit()
|
|
161
|
-
assert phrasplit_sentences is not None
|
|
162
|
-
|
|
163
103
|
paragraphs = split_paragraphs(text)
|
|
164
104
|
|
|
165
105
|
if not paragraphs:
|
|
@@ -180,7 +120,7 @@ def format_sentences(
|
|
|
180
120
|
continue
|
|
181
121
|
|
|
182
122
|
# Process paragraph into sentences using phrasplit
|
|
183
|
-
sentences =
|
|
123
|
+
sentences = split_sentences(para, language_model)
|
|
184
124
|
|
|
185
125
|
if not sentences:
|
|
186
126
|
continue
|
|
@@ -224,8 +164,6 @@ def format_clauses(
|
|
|
224
164
|
"I do like coffee,
|
|
225
165
|
and I like wine."
|
|
226
166
|
"""
|
|
227
|
-
_check_phrasplit()
|
|
228
|
-
assert phrasplit_clauses is not None
|
|
229
167
|
|
|
230
168
|
paragraphs = split_paragraphs(text)
|
|
231
169
|
|
|
@@ -246,7 +184,7 @@ def format_clauses(
|
|
|
246
184
|
continue
|
|
247
185
|
|
|
248
186
|
# Process paragraph into clauses using phrasplit
|
|
249
|
-
clauses =
|
|
187
|
+
clauses = split_clauses(para, language_model)
|
|
250
188
|
|
|
251
189
|
if not clauses:
|
|
252
190
|
continue
|
|
@@ -277,8 +215,6 @@ def split_long_lines(
|
|
|
277
215
|
2. If still too long, split at clause boundaries (commas, semicolons, etc.)
|
|
278
216
|
3. If still too long, split at word boundaries
|
|
279
217
|
|
|
280
|
-
Requires phrasplit to be installed.
|
|
281
|
-
|
|
282
218
|
Args:
|
|
283
219
|
text: Input text (may already be formatted)
|
|
284
220
|
max_length: Maximum line length in characters
|
|
@@ -288,9 +224,6 @@ def split_long_lines(
|
|
|
288
224
|
Returns:
|
|
289
225
|
Text with long lines split. Chapter titles are preserved.
|
|
290
226
|
"""
|
|
291
|
-
_check_phrasplit()
|
|
292
|
-
assert phrasplit_long_lines is not None
|
|
293
|
-
|
|
294
227
|
lines = text.split("\n")
|
|
295
228
|
result_lines: list[str] = []
|
|
296
229
|
|
|
@@ -315,9 +248,13 @@ def split_long_lines(
|
|
|
315
248
|
has_separator = line.startswith(separator) if separator else False
|
|
316
249
|
content = line[len(separator) :] if has_separator else line
|
|
317
250
|
|
|
318
|
-
# Split the long line using phrasplit
|
|
319
|
-
split_lines_list =
|
|
320
|
-
|
|
251
|
+
# Split the long line using phrasplit (avoid calling ourselves)
|
|
252
|
+
split_lines_list = _phrasplit_split_long_lines(
|
|
253
|
+
content, max_length, language_model
|
|
254
|
+
)
|
|
255
|
+
# Defensive: if upstream ever returns a string, normalize to list
|
|
256
|
+
if isinstance(split_lines_list, str):
|
|
257
|
+
split_lines_list = [split_lines_list]
|
|
321
258
|
# Add separator to first line if original had it
|
|
322
259
|
for k, split_line in enumerate(split_lines_list):
|
|
323
260
|
if k == 0 and has_separator:
|
|
@@ -328,32 +265,11 @@ def split_long_lines(
|
|
|
328
265
|
return "\n".join(result_lines)
|
|
329
266
|
|
|
330
267
|
|
|
331
|
-
# Keep backward compatibility alias
|
|
332
|
-
def format_as_sentences(text: str, language_model: str = "en_core_web_sm") -> str:
|
|
333
|
-
"""
|
|
334
|
-
Format text with one sentence per line using spaCy.
|
|
335
|
-
|
|
336
|
-
Deprecated: Use format_sentences() instead.
|
|
337
|
-
|
|
338
|
-
Requires phrasplit to be installed.
|
|
339
|
-
|
|
340
|
-
Args:
|
|
341
|
-
text: Input text with paragraph breaks
|
|
342
|
-
language_model: spaCy language model to use (default: "en_core_web_sm")
|
|
343
|
-
|
|
344
|
-
Returns:
|
|
345
|
-
Text with sentences separated by newlines
|
|
346
|
-
"""
|
|
347
|
-
return format_sentences(text, separator="", language_model=language_model)
|
|
348
|
-
|
|
349
|
-
|
|
350
268
|
__all__ = [
|
|
351
|
-
"PHRASPLIT_AVAILABLE",
|
|
352
269
|
"collapse_paragraph",
|
|
353
270
|
"format_paragraphs",
|
|
354
271
|
"format_sentences",
|
|
355
272
|
"format_clauses",
|
|
356
273
|
"split_long_lines",
|
|
357
|
-
"format_as_sentences",
|
|
358
274
|
"split_paragraphs",
|
|
359
275
|
]
|
|
@@ -57,9 +57,25 @@ class EPUBParser:
|
|
|
57
57
|
self._metadata: Optional[Metadata] = None
|
|
58
58
|
self._page_list_nav: Optional[tuple[Any, str]] = None
|
|
59
59
|
self._page_list_nav_checked = False
|
|
60
|
+
self._nav_processed = False
|
|
61
|
+
self._upper_title_cache: Optional[set[str]] = None
|
|
60
62
|
|
|
61
63
|
self._load_epub()
|
|
62
64
|
|
|
65
|
+
def _get_upper_titles_for_toc(self) -> set[str]:
|
|
66
|
+
if self._upper_title_cache is not None:
|
|
67
|
+
return self._upper_title_cache
|
|
68
|
+
titles: set[str] = set()
|
|
69
|
+
meta = self.get_metadata()
|
|
70
|
+
if meta.title:
|
|
71
|
+
titles.add(meta.title.upper())
|
|
72
|
+
# IMPORTANT: no chapter text allocation here
|
|
73
|
+
for ch in self.get_chapters(include_text=False):
|
|
74
|
+
if ch.title:
|
|
75
|
+
titles.add(ch.title.upper())
|
|
76
|
+
self._upper_title_cache = titles
|
|
77
|
+
return titles
|
|
78
|
+
|
|
63
79
|
def _load_epub(self) -> None:
|
|
64
80
|
"""Load and parse the EPUB file."""
|
|
65
81
|
try:
|
|
@@ -133,7 +149,8 @@ class EPUBParser:
|
|
|
133
149
|
"""
|
|
134
150
|
# First, process the navigation structure
|
|
135
151
|
self._process_epub_content_nav()
|
|
136
|
-
|
|
152
|
+
if not self._nav_processed:
|
|
153
|
+
self._process_epub_content_nav()
|
|
137
154
|
# Convert the navigation structure to Chapter objects
|
|
138
155
|
chapters: list[Chapter] = []
|
|
139
156
|
self._build_chapters_from_nav(
|
|
@@ -201,6 +218,8 @@ class EPUBParser:
|
|
|
201
218
|
Process EPUB content using ITEM_NAVIGATION (NAV HTML) or ITEM_NCX.
|
|
202
219
|
Globally orders navigation entries and slices content between them.
|
|
203
220
|
"""
|
|
221
|
+
if self._nav_processed:
|
|
222
|
+
return
|
|
204
223
|
logger.info("Processing EPUB using navigation document (NAV/NCX)...")
|
|
205
224
|
nav_item = None
|
|
206
225
|
nav_type = None
|
|
@@ -483,6 +502,8 @@ class EPUBParser:
|
|
|
483
502
|
logger.info("Added prefix content chapter")
|
|
484
503
|
|
|
485
504
|
logger.info(f"Finished processing. Found {len(self.content_texts)} sections")
|
|
505
|
+
self._nav_processed = True
|
|
506
|
+
self._upper_title_cache = None # invalidate if reprocessed for any reason
|
|
486
507
|
|
|
487
508
|
def _find_doc_key(
|
|
488
509
|
self,
|
|
@@ -817,6 +838,7 @@ class EPUBParser:
|
|
|
817
838
|
chapter_ids: Optional[list[str]] = None,
|
|
818
839
|
deduplicate_chapter_titles: bool = True,
|
|
819
840
|
skip_toc: bool = False,
|
|
841
|
+
include_chapter_title: bool = False,
|
|
820
842
|
) -> str:
|
|
821
843
|
"""
|
|
822
844
|
Extract text from selected chapters.
|
|
@@ -826,6 +848,8 @@ class EPUBParser:
|
|
|
826
848
|
deduplicate_chapter_titles: If True, removes duplicate chapter titles
|
|
827
849
|
that appear as the first line of chapter content (default: True)
|
|
828
850
|
skip_toc: If True, skip TOC and front matter chapters (default: False)
|
|
851
|
+
include_chapter_title: If True, includes chapter titles in the output
|
|
852
|
+
(default: False)
|
|
829
853
|
|
|
830
854
|
Returns:
|
|
831
855
|
Combined text from all selected chapters with chapter titles separated
|
|
@@ -870,12 +894,14 @@ class EPUBParser:
|
|
|
870
894
|
|
|
871
895
|
# Format: 4 linebreaks (if not first), chapter title,
|
|
872
896
|
# 2 linebreaks, content
|
|
873
|
-
if i == 0:
|
|
897
|
+
if i == 0 and include_chapter_title:
|
|
874
898
|
# First chapter: no leading linebreaks
|
|
875
899
|
parts.append(f"{chapter.title}\n\n{cleaned_text}")
|
|
876
|
-
|
|
900
|
+
elif i > 0 and include_chapter_title:
|
|
877
901
|
# Subsequent chapters: 4 linebreaks before title
|
|
878
902
|
parts.append(f"\n\n\n\n{chapter.title}\n\n{cleaned_text}")
|
|
903
|
+
else:
|
|
904
|
+
parts.append(cleaned_text)
|
|
879
905
|
|
|
880
906
|
return "".join(parts)
|
|
881
907
|
|
|
@@ -1242,9 +1268,16 @@ class EPUBParser:
|
|
|
1242
1268
|
|
|
1243
1269
|
return pages
|
|
1244
1270
|
|
|
1245
|
-
def _split_into_sentences(
|
|
1271
|
+
def _split_into_sentences(
|
|
1272
|
+
self,
|
|
1273
|
+
text: str,
|
|
1274
|
+
language_model: str = "en_core_web_sm",
|
|
1275
|
+
apply_corrections: bool = True,
|
|
1276
|
+
split_on_colon: bool = True,
|
|
1277
|
+
use_spacy: bool = False,
|
|
1278
|
+
) -> list[str]:
|
|
1246
1279
|
"""
|
|
1247
|
-
Split text into sentences
|
|
1280
|
+
Split text into sentences, preserving paragraph breaks.
|
|
1248
1281
|
|
|
1249
1282
|
Handles common abbreviations and edge cases to avoid incorrect splits.
|
|
1250
1283
|
Paragraph breaks (\\n\\n) are preserved within the sentence list.
|
|
@@ -1258,106 +1291,26 @@ class EPUBParser:
|
|
|
1258
1291
|
if not text or not text.strip():
|
|
1259
1292
|
return []
|
|
1260
1293
|
|
|
1261
|
-
|
|
1294
|
+
from phrasplit import split_paragraphs, split_sentences
|
|
1262
1295
|
|
|
1263
|
-
|
|
1264
|
-
paragraphs = text
|
|
1296
|
+
sentences: list[str] = []
|
|
1297
|
+
paragraphs = split_paragraphs(text)
|
|
1265
1298
|
|
|
1266
1299
|
for para_idx, paragraph in enumerate(paragraphs):
|
|
1267
1300
|
if not paragraph.strip():
|
|
1268
1301
|
continue
|
|
1269
1302
|
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
"Sr",
|
|
1282
|
-
"Jr",
|
|
1283
|
-
"vs",
|
|
1284
|
-
"etc",
|
|
1285
|
-
"Inc",
|
|
1286
|
-
"Ltd",
|
|
1287
|
-
"Corp",
|
|
1288
|
-
"Co",
|
|
1289
|
-
"St",
|
|
1290
|
-
"Ave",
|
|
1291
|
-
"Blvd",
|
|
1292
|
-
"Rd",
|
|
1293
|
-
"Fig",
|
|
1294
|
-
"No",
|
|
1295
|
-
"Vol",
|
|
1296
|
-
"pp",
|
|
1297
|
-
"Jan",
|
|
1298
|
-
"Feb",
|
|
1299
|
-
"Mar",
|
|
1300
|
-
"Apr",
|
|
1301
|
-
"Jun",
|
|
1302
|
-
"Jul",
|
|
1303
|
-
"Aug",
|
|
1304
|
-
"Sep",
|
|
1305
|
-
"Oct",
|
|
1306
|
-
"Nov",
|
|
1307
|
-
"Dec",
|
|
1308
|
-
"e.g",
|
|
1309
|
-
"i.e",
|
|
1310
|
-
}
|
|
1311
|
-
|
|
1312
|
-
current_sentence: list[str] = []
|
|
1313
|
-
|
|
1314
|
-
# Split by whitespace and process word by word
|
|
1315
|
-
words = paragraph.split()
|
|
1316
|
-
|
|
1317
|
-
for i, word in enumerate(words):
|
|
1318
|
-
current_sentence.append(word)
|
|
1319
|
-
|
|
1320
|
-
# Check if this word ends a sentence
|
|
1321
|
-
if word and word[-1] in ".!?":
|
|
1322
|
-
# Check if it's an abbreviation
|
|
1323
|
-
word_without_punct = word.rstrip(".!?")
|
|
1324
|
-
is_abbreviation = word_without_punct in abbreviations
|
|
1325
|
-
|
|
1326
|
-
# Check for ellipsis (... or . . .)
|
|
1327
|
-
is_ellipsis = word == "..." or (
|
|
1328
|
-
word == "."
|
|
1329
|
-
and i >= 2
|
|
1330
|
-
and words[i - 1] == "."
|
|
1331
|
-
and words[i - 2] == "."
|
|
1332
|
-
)
|
|
1333
|
-
|
|
1334
|
-
# Check if next word starts with capital (if exists)
|
|
1335
|
-
next_word_capital = False
|
|
1336
|
-
if i + 1 < len(words):
|
|
1337
|
-
next_word = words[i + 1]
|
|
1338
|
-
if next_word and next_word[0].isupper():
|
|
1339
|
-
next_word_capital = True
|
|
1340
|
-
|
|
1341
|
-
# End sentence if:
|
|
1342
|
-
# - Not an abbreviation
|
|
1343
|
-
# - Not an ellipsis
|
|
1344
|
-
# - Either at end of text or next word starts with capital
|
|
1345
|
-
if (
|
|
1346
|
-
not is_abbreviation
|
|
1347
|
-
and not is_ellipsis
|
|
1348
|
-
and (i + 1 >= len(words) or next_word_capital)
|
|
1349
|
-
):
|
|
1350
|
-
sentence = " ".join(current_sentence)
|
|
1351
|
-
if sentence.strip():
|
|
1352
|
-
sentences.append(sentence.strip())
|
|
1353
|
-
current_sentence = []
|
|
1354
|
-
|
|
1355
|
-
# Add any remaining text as final sentence
|
|
1356
|
-
if current_sentence:
|
|
1357
|
-
sentence = " ".join(current_sentence)
|
|
1358
|
-
if sentence.strip():
|
|
1359
|
-
sentences.append(sentence.strip())
|
|
1360
|
-
|
|
1303
|
+
para_sentences = split_sentences(
|
|
1304
|
+
paragraph,
|
|
1305
|
+
language_model=language_model,
|
|
1306
|
+
apply_corrections=apply_corrections,
|
|
1307
|
+
split_on_colon=split_on_colon,
|
|
1308
|
+
use_spacy=use_spacy,
|
|
1309
|
+
)
|
|
1310
|
+
for sent in para_sentences:
|
|
1311
|
+
sent = sent.strip()
|
|
1312
|
+
if sent:
|
|
1313
|
+
sentences.append(sent)
|
|
1361
1314
|
# Add paragraph break marker after each paragraph (except the last)
|
|
1362
1315
|
if para_idx < len(paragraphs) - 1:
|
|
1363
1316
|
sentences.append("\n\n") # Paragraph break marker
|
|
@@ -1665,20 +1618,7 @@ class EPUBParser:
|
|
|
1665
1618
|
if page.chapter_title and page.chapter_title.upper() in front_matter_keywords:
|
|
1666
1619
|
return True
|
|
1667
1620
|
|
|
1668
|
-
|
|
1669
|
-
search_terms = []
|
|
1670
|
-
|
|
1671
|
-
# Add book title
|
|
1672
|
-
metadata = self.get_metadata()
|
|
1673
|
-
if metadata.title:
|
|
1674
|
-
search_terms.append(metadata.title.upper())
|
|
1675
|
-
|
|
1676
|
-
# Add all chapter titles
|
|
1677
|
-
chapters = self.get_chapters()
|
|
1678
|
-
for chapter in chapters:
|
|
1679
|
-
if chapter.title:
|
|
1680
|
-
search_terms.append(chapter.title.upper())
|
|
1681
|
-
|
|
1621
|
+
search_terms = list(self._get_upper_titles_for_toc())
|
|
1682
1622
|
if not search_terms:
|
|
1683
1623
|
return False
|
|
1684
1624
|
|
|
@@ -1716,9 +1656,7 @@ class EPUBParser:
|
|
|
1716
1656
|
return page_text
|
|
1717
1657
|
|
|
1718
1658
|
# Build list of chapter titles to detect TOC
|
|
1719
|
-
|
|
1720
|
-
chapter_titles = {ch.title.upper() for ch in chapters if ch.title}
|
|
1721
|
-
|
|
1659
|
+
chapter_titles = self._get_upper_titles_for_toc()
|
|
1722
1660
|
if not chapter_titles:
|
|
1723
1661
|
return page_text
|
|
1724
1662
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: epub2text
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.3
|
|
4
4
|
Summary: A simple tool to extract text from EPUB files.
|
|
5
5
|
Author-email: Holger Nahrstaedt <nahrstaedt@gmail.com>
|
|
6
6
|
License: MIT License
|
|
@@ -45,10 +45,9 @@ Requires-Dist: rich>=13.0.0
|
|
|
45
45
|
Requires-Dist: ebooklib>=0.18
|
|
46
46
|
Requires-Dist: beautifulsoup4>=4.12.0
|
|
47
47
|
Requires-Dist: defusedxml>=0.7.1
|
|
48
|
+
Requires-Dist: phrasplit>=0.3.2
|
|
48
49
|
Provides-Extra: lxml
|
|
49
50
|
Requires-Dist: lxml>=4.9.0; extra == "lxml"
|
|
50
|
-
Provides-Extra: sentences
|
|
51
|
-
Requires-Dist: phrasplit>=0.1.0; extra == "sentences"
|
|
52
51
|
Provides-Extra: dev
|
|
53
52
|
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
54
53
|
Requires-Dist: ruff>=0.1.0; extra == "dev"
|
|
@@ -30,7 +30,6 @@ from epub2text import (
|
|
|
30
30
|
epub2txt,
|
|
31
31
|
)
|
|
32
32
|
from epub2text.formatters import (
|
|
33
|
-
PHRASPLIT_AVAILABLE,
|
|
34
33
|
collapse_paragraph,
|
|
35
34
|
format_clauses,
|
|
36
35
|
format_paragraphs,
|
|
@@ -549,11 +548,6 @@ def example_formatters_sentences() -> None:
|
|
|
549
548
|
print("EXAMPLE: Sentence Formatting")
|
|
550
549
|
print("=" * 60)
|
|
551
550
|
|
|
552
|
-
if not PHRASPLIT_AVAILABLE:
|
|
553
|
-
print("\nNote: Install phrasplit for sentence formatting:")
|
|
554
|
-
print(" pip install epub2text[sentences]")
|
|
555
|
-
return
|
|
556
|
-
|
|
557
551
|
text = """Dr. Smith went to Washington D.C. for a meeting. She arrived
|
|
558
552
|
at 3 p.m. and stayed until evening. The conference was very productive.
|
|
559
553
|
|
|
@@ -579,11 +573,6 @@ def example_formatters_clauses() -> None:
|
|
|
579
573
|
print("EXAMPLE: Clause Formatting")
|
|
580
574
|
print("=" * 60)
|
|
581
575
|
|
|
582
|
-
if not PHRASPLIT_AVAILABLE:
|
|
583
|
-
print("\nNote: Install phrasplit for clause formatting:")
|
|
584
|
-
print(" pip install epub2text[sentences]")
|
|
585
|
-
return
|
|
586
|
-
|
|
587
576
|
text = """When the sun rises, the birds begin to sing, and the world slowly awakens.
|
|
588
577
|
|
|
589
578
|
Complex sentences, like this one, benefit from clause splitting,
|
|
@@ -609,11 +598,6 @@ def example_formatters_long_lines() -> None:
|
|
|
609
598
|
print("EXAMPLE: Long Line Splitting")
|
|
610
599
|
print("=" * 60)
|
|
611
600
|
|
|
612
|
-
if not PHRASPLIT_AVAILABLE:
|
|
613
|
-
print("\nNote: Install phrasplit for long line splitting:")
|
|
614
|
-
print(" pip install epub2text[sentences]")
|
|
615
|
-
return
|
|
616
|
-
|
|
617
601
|
long_text = (
|
|
618
602
|
"This is an extremely long line that contains multiple sentences "
|
|
619
603
|
"and clauses, which would be difficult to read on a narrow display, "
|
|
@@ -22,6 +22,7 @@ dependencies = [
|
|
|
22
22
|
"ebooklib>=0.18",
|
|
23
23
|
"beautifulsoup4>=4.12.0",
|
|
24
24
|
"defusedxml>=0.7.1",
|
|
25
|
+
"phrasplit>=0.3.2",
|
|
25
26
|
]
|
|
26
27
|
classifiers=[
|
|
27
28
|
'Intended Audience :: Developers',
|
|
@@ -43,9 +44,6 @@ dynamic=["version"]
|
|
|
43
44
|
lxml = [
|
|
44
45
|
"lxml>=4.9.0",
|
|
45
46
|
]
|
|
46
|
-
sentences = [
|
|
47
|
-
"phrasplit>=0.1.0",
|
|
48
|
-
]
|
|
49
47
|
dev = [
|
|
50
48
|
"pytest>=7.0.0",
|
|
51
49
|
"ruff>=0.1.0",
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
+
import contextlib
|
|
5
6
|
import threading
|
|
6
7
|
import time
|
|
7
8
|
from collections.abc import Generator
|
|
@@ -43,8 +44,12 @@ def serve_directory(
|
|
|
43
44
|
host, port = server.server_address[:2]
|
|
44
45
|
yield f"http://{host}:{port}"
|
|
45
46
|
finally:
|
|
46
|
-
|
|
47
|
-
|
|
47
|
+
# Stop the loop, then close the listening socket to avoid FD/memory growth
|
|
48
|
+
with contextlib.suppress(Exception):
|
|
49
|
+
server.shutdown()
|
|
50
|
+
thread.join(timeout=5)
|
|
51
|
+
with contextlib.suppress(Exception):
|
|
52
|
+
server.server_close()
|
|
48
53
|
|
|
49
54
|
|
|
50
55
|
def _create_epub(epub_path: Path) -> None:
|
|
@@ -13,7 +13,6 @@ import pypub # type: ignore[import-untyped]
|
|
|
13
13
|
import pytest
|
|
14
14
|
|
|
15
15
|
from epub2text import EPUBParser, epub2txt
|
|
16
|
-
from epub2text.formatters import PHRASPLIT_AVAILABLE
|
|
17
16
|
|
|
18
17
|
# Check if spaCy language model is available
|
|
19
18
|
try:
|
|
@@ -29,12 +28,8 @@ except ImportError:
|
|
|
29
28
|
SPACY_AVAILABLE = False
|
|
30
29
|
SPACY_MODEL_AVAILABLE = False
|
|
31
30
|
|
|
32
|
-
requires_phrasplit = pytest.mark.skipif(
|
|
33
|
-
not PHRASPLIT_AVAILABLE,
|
|
34
|
-
reason="phrasplit required. Install with: pip install epub2text[sentences]",
|
|
35
|
-
)
|
|
36
31
|
requires_spacy_model = pytest.mark.skipif(
|
|
37
|
-
not
|
|
32
|
+
not SPACY_MODEL_AVAILABLE,
|
|
38
33
|
reason=(
|
|
39
34
|
"phrasplit + spaCy model required. Install with: pip install "
|
|
40
35
|
"epub2text[sentences] and python -m spacy download en_core_web_sm"
|
|
@@ -959,17 +954,16 @@ class TestMetadataExtraction:
|
|
|
959
954
|
class TestFormatters:
|
|
960
955
|
"""Test text formatting functions."""
|
|
961
956
|
|
|
962
|
-
@requires_phrasplit
|
|
963
957
|
def test_split_at_clauses_preserves_all_text(self) -> None:
|
|
964
958
|
"""Test that _split_at_clauses doesn't lose any text."""
|
|
965
|
-
from phrasplit
|
|
959
|
+
from phrasplit import split_clauses
|
|
966
960
|
|
|
967
961
|
text = (
|
|
968
962
|
"When the professor picked up the ancient manuscript for the "
|
|
969
963
|
"ninth time, Sarah whispered, 'You will find the answer in the margins.'"
|
|
970
964
|
)
|
|
971
965
|
|
|
972
|
-
result =
|
|
966
|
+
result = split_clauses(text)
|
|
973
967
|
joined = " ".join(result)
|
|
974
968
|
|
|
975
969
|
# All words from original should be in result
|
|
@@ -979,17 +973,16 @@ class TestMetadataExtraction:
|
|
|
979
973
|
assert "Sarah whispered" in joined
|
|
980
974
|
assert "margins" in joined
|
|
981
975
|
|
|
982
|
-
@requires_phrasplit
|
|
983
976
|
def test_split_at_clauses_no_trailing_comma(self) -> None:
|
|
984
977
|
"""Test that _split_at_clauses doesn't produce lines ending with just comma."""
|
|
985
|
-
from phrasplit
|
|
978
|
+
from phrasplit import split_clauses
|
|
986
979
|
|
|
987
980
|
text = (
|
|
988
981
|
"When the professor picked up the ancient manuscript for the "
|
|
989
982
|
"ninth time, Sarah whispered, 'You will find the answer in the margins.'"
|
|
990
983
|
)
|
|
991
984
|
|
|
992
|
-
result =
|
|
985
|
+
result = split_clauses(text)
|
|
993
986
|
|
|
994
987
|
# No line should end with just a comma (indicates lost text)
|
|
995
988
|
for line in result:
|
|
@@ -211,7 +211,9 @@ class TestTitleDeduplication:
|
|
|
211
211
|
parser = EPUBParser(str(epub_with_duplicate_titles))
|
|
212
212
|
|
|
213
213
|
# Extract with deduplication (default)
|
|
214
|
-
text = parser.extract_chapters(
|
|
214
|
+
text = parser.extract_chapters(
|
|
215
|
+
deduplicate_chapter_titles=True, include_chapter_title=True
|
|
216
|
+
)
|
|
215
217
|
|
|
216
218
|
# Chapter should have title in new format (preceded by 4 newlines
|
|
217
219
|
# or at start, followed by 2 newlines)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|