phrasplit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- phrasplit/__init__.py +21 -0
- phrasplit/_version.py +34 -0
- phrasplit/cli.py +207 -0
- phrasplit/py.typed +0 -0
- phrasplit/splitter.py +345 -0
- phrasplit-0.1.0.dist-info/METADATA +227 -0
- phrasplit-0.1.0.dist-info/RECORD +11 -0
- phrasplit-0.1.0.dist-info/WHEEL +5 -0
- phrasplit-0.1.0.dist-info/entry_points.txt +2 -0
- phrasplit-0.1.0.dist-info/licenses/LICENSE +21 -0
- phrasplit-0.1.0.dist-info/top_level.txt +1 -0
phrasplit/__init__.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""Phrasplit - Split text into sentences, clauses, or paragraphs."""
|
|
2
|
+
|
|
3
|
+
from .splitter import (
|
|
4
|
+
split_clauses,
|
|
5
|
+
split_long_lines,
|
|
6
|
+
split_paragraphs,
|
|
7
|
+
split_sentences,
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
try:
|
|
11
|
+
from ._version import version as __version__
|
|
12
|
+
except ImportError:
|
|
13
|
+
__version__ = "0.0.0"
|
|
14
|
+
|
|
15
|
+
__all__ = [
|
|
16
|
+
"__version__",
|
|
17
|
+
"split_clauses",
|
|
18
|
+
"split_long_lines",
|
|
19
|
+
"split_paragraphs",
|
|
20
|
+
"split_sentences",
|
|
21
|
+
]
|
phrasplit/_version.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# file generated by setuptools-scm
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
|
|
4
|
+
__all__ = [
|
|
5
|
+
"__version__",
|
|
6
|
+
"__version_tuple__",
|
|
7
|
+
"version",
|
|
8
|
+
"version_tuple",
|
|
9
|
+
"__commit_id__",
|
|
10
|
+
"commit_id",
|
|
11
|
+
]
|
|
12
|
+
|
|
13
|
+
TYPE_CHECKING = False
|
|
14
|
+
if TYPE_CHECKING:
|
|
15
|
+
from typing import Tuple
|
|
16
|
+
from typing import Union
|
|
17
|
+
|
|
18
|
+
VERSION_TUPLE = Tuple[Union[int, str], ...]
|
|
19
|
+
COMMIT_ID = Union[str, None]
|
|
20
|
+
else:
|
|
21
|
+
VERSION_TUPLE = object
|
|
22
|
+
COMMIT_ID = object
|
|
23
|
+
|
|
24
|
+
version: str
|
|
25
|
+
__version__: str
|
|
26
|
+
__version_tuple__: VERSION_TUPLE
|
|
27
|
+
version_tuple: VERSION_TUPLE
|
|
28
|
+
commit_id: COMMIT_ID
|
|
29
|
+
__commit_id__: COMMIT_ID
|
|
30
|
+
|
|
31
|
+
__version__ = version = '0.1.0'
|
|
32
|
+
__version_tuple__ = version_tuple = (0, 1, 0)
|
|
33
|
+
|
|
34
|
+
__commit_id__ = commit_id = None
|
phrasplit/cli.py
ADDED
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
"""Command-line interface for phrasplit."""
|
|
2
|
+
|
|
3
|
+
import sys
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Optional
|
|
6
|
+
|
|
7
|
+
import click
|
|
8
|
+
from rich.console import Console
|
|
9
|
+
|
|
10
|
+
from .splitter import split_clauses, split_long_lines, split_paragraphs, split_sentences
|
|
11
|
+
|
|
12
|
+
console = Console()
|
|
13
|
+
error_console = Console(stderr=True)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def read_input(input_file: Optional[str]) -> str:
|
|
17
|
+
"""Read input from file or stdin.
|
|
18
|
+
|
|
19
|
+
Args:
|
|
20
|
+
input_file: Path to input file, '-' for stdin, or None for stdin
|
|
21
|
+
|
|
22
|
+
Returns:
|
|
23
|
+
Text content
|
|
24
|
+
"""
|
|
25
|
+
if input_file is None or input_file == "-":
|
|
26
|
+
return sys.stdin.read()
|
|
27
|
+
return Path(input_file).read_text(encoding="utf-8")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def write_output(text: str, output: Optional[Path], use_rich: bool = True) -> None:
|
|
31
|
+
"""Write output to file or stdout.
|
|
32
|
+
|
|
33
|
+
Args:
|
|
34
|
+
text: Text to write
|
|
35
|
+
output: Output file path or None for stdout
|
|
36
|
+
use_rich: Whether to use rich console for stdout
|
|
37
|
+
"""
|
|
38
|
+
if output:
|
|
39
|
+
output.write_text(text, encoding="utf-8")
|
|
40
|
+
error_console.print(f"[green]Output written to {output}[/green]")
|
|
41
|
+
elif use_rich:
|
|
42
|
+
console.print(text)
|
|
43
|
+
else:
|
|
44
|
+
print(text)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@click.group()
|
|
48
|
+
@click.version_option()
|
|
49
|
+
def main() -> None:
|
|
50
|
+
"""Phrasplit - Split text into sentences, clauses, or paragraphs."""
|
|
51
|
+
pass
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@main.command()
|
|
55
|
+
@click.argument("input_file", required=False, default=None)
|
|
56
|
+
@click.option(
|
|
57
|
+
"-o",
|
|
58
|
+
"--output",
|
|
59
|
+
type=click.Path(path_type=Path),
|
|
60
|
+
help="Output file (default: stdout)",
|
|
61
|
+
)
|
|
62
|
+
@click.option(
|
|
63
|
+
"-m",
|
|
64
|
+
"--model",
|
|
65
|
+
default="en_core_web_sm",
|
|
66
|
+
help="spaCy language model (default: en_core_web_sm)",
|
|
67
|
+
)
|
|
68
|
+
def sentences(
|
|
69
|
+
input_file: Optional[str],
|
|
70
|
+
output: Optional[Path],
|
|
71
|
+
model: str,
|
|
72
|
+
) -> None:
|
|
73
|
+
"""Split text into sentences.
|
|
74
|
+
|
|
75
|
+
INPUT_FILE: Path to input file, or '-' for stdin. Reads from stdin if omitted.
|
|
76
|
+
"""
|
|
77
|
+
try:
|
|
78
|
+
text = read_input(input_file)
|
|
79
|
+
except FileNotFoundError:
|
|
80
|
+
error_console.print(f"[red]Error:[/red] File not found: {input_file}")
|
|
81
|
+
sys.exit(1)
|
|
82
|
+
|
|
83
|
+
try:
|
|
84
|
+
result = split_sentences(text, language_model=model)
|
|
85
|
+
except (ImportError, OSError) as e:
|
|
86
|
+
error_console.print(f"[red]Error:[/red] {e}")
|
|
87
|
+
sys.exit(1)
|
|
88
|
+
|
|
89
|
+
output_text = "\n".join(result)
|
|
90
|
+
write_output(output_text, output)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
@main.command()
|
|
94
|
+
@click.argument("input_file", required=False, default=None)
|
|
95
|
+
@click.option(
|
|
96
|
+
"-o",
|
|
97
|
+
"--output",
|
|
98
|
+
type=click.Path(path_type=Path),
|
|
99
|
+
help="Output file (default: stdout)",
|
|
100
|
+
)
|
|
101
|
+
@click.option(
|
|
102
|
+
"-m",
|
|
103
|
+
"--model",
|
|
104
|
+
default="en_core_web_sm",
|
|
105
|
+
help="spaCy language model (default: en_core_web_sm)",
|
|
106
|
+
)
|
|
107
|
+
def clauses(
|
|
108
|
+
input_file: Optional[str],
|
|
109
|
+
output: Optional[Path],
|
|
110
|
+
model: str,
|
|
111
|
+
) -> None:
|
|
112
|
+
"""Split text into clauses (at commas).
|
|
113
|
+
|
|
114
|
+
INPUT_FILE: Path to input file, or '-' for stdin. Reads from stdin if omitted.
|
|
115
|
+
"""
|
|
116
|
+
try:
|
|
117
|
+
text = read_input(input_file)
|
|
118
|
+
except FileNotFoundError:
|
|
119
|
+
error_console.print(f"[red]Error:[/red] File not found: {input_file}")
|
|
120
|
+
sys.exit(1)
|
|
121
|
+
|
|
122
|
+
try:
|
|
123
|
+
result = split_clauses(text, language_model=model)
|
|
124
|
+
except (ImportError, OSError) as e:
|
|
125
|
+
error_console.print(f"[red]Error:[/red] {e}")
|
|
126
|
+
sys.exit(1)
|
|
127
|
+
|
|
128
|
+
output_text = "\n".join(result)
|
|
129
|
+
write_output(output_text, output)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
@main.command()
|
|
133
|
+
@click.argument("input_file", required=False, default=None)
|
|
134
|
+
@click.option(
|
|
135
|
+
"-o",
|
|
136
|
+
"--output",
|
|
137
|
+
type=click.Path(path_type=Path),
|
|
138
|
+
help="Output file (default: stdout)",
|
|
139
|
+
)
|
|
140
|
+
def paragraphs(
|
|
141
|
+
input_file: Optional[str],
|
|
142
|
+
output: Optional[Path],
|
|
143
|
+
) -> None:
|
|
144
|
+
"""Split text into paragraphs.
|
|
145
|
+
|
|
146
|
+
INPUT_FILE: Path to input file, or '-' for stdin. Reads from stdin if omitted.
|
|
147
|
+
"""
|
|
148
|
+
try:
|
|
149
|
+
text = read_input(input_file)
|
|
150
|
+
except FileNotFoundError:
|
|
151
|
+
error_console.print(f"[red]Error:[/red] File not found: {input_file}")
|
|
152
|
+
sys.exit(1)
|
|
153
|
+
|
|
154
|
+
result = split_paragraphs(text)
|
|
155
|
+
output_text = "\n\n".join(result)
|
|
156
|
+
write_output(output_text, output)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
@main.command()
|
|
160
|
+
@click.argument("input_file", required=False, default=None)
|
|
161
|
+
@click.option(
|
|
162
|
+
"-o",
|
|
163
|
+
"--output",
|
|
164
|
+
type=click.Path(path_type=Path),
|
|
165
|
+
help="Output file (default: stdout)",
|
|
166
|
+
)
|
|
167
|
+
@click.option(
|
|
168
|
+
"-l",
|
|
169
|
+
"--max-length",
|
|
170
|
+
default=80,
|
|
171
|
+
type=int,
|
|
172
|
+
help="Maximum line length (default: 80)",
|
|
173
|
+
)
|
|
174
|
+
@click.option(
|
|
175
|
+
"-m",
|
|
176
|
+
"--model",
|
|
177
|
+
default="en_core_web_sm",
|
|
178
|
+
help="spaCy language model (default: en_core_web_sm)",
|
|
179
|
+
)
|
|
180
|
+
def longlines(
|
|
181
|
+
input_file: Optional[str],
|
|
182
|
+
output: Optional[Path],
|
|
183
|
+
max_length: int,
|
|
184
|
+
model: str,
|
|
185
|
+
) -> None:
|
|
186
|
+
"""Split long lines at sentence/clause boundaries.
|
|
187
|
+
|
|
188
|
+
INPUT_FILE: Path to input file, or '-' for stdin. Reads from stdin if omitted.
|
|
189
|
+
"""
|
|
190
|
+
try:
|
|
191
|
+
text = read_input(input_file)
|
|
192
|
+
except FileNotFoundError:
|
|
193
|
+
error_console.print(f"[red]Error:[/red] File not found: {input_file}")
|
|
194
|
+
sys.exit(1)
|
|
195
|
+
|
|
196
|
+
try:
|
|
197
|
+
result = split_long_lines(text, max_length=max_length, language_model=model)
|
|
198
|
+
except (ImportError, OSError) as e:
|
|
199
|
+
error_console.print(f"[red]Error:[/red] {e}")
|
|
200
|
+
sys.exit(1)
|
|
201
|
+
|
|
202
|
+
output_text = "\n".join(result)
|
|
203
|
+
write_output(output_text, output)
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
if __name__ == "__main__":
|
|
207
|
+
main()
|
phrasplit/py.typed
ADDED
|
File without changes
|
phrasplit/splitter.py
ADDED
|
@@ -0,0 +1,345 @@
|
|
|
1
|
+
"""Text splitting utilities using spaCy for NLP-based sentence and clause detection."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
try:
|
|
7
|
+
import spacy # type: ignore[import-not-found]
|
|
8
|
+
|
|
9
|
+
SPACY_AVAILABLE = True
|
|
10
|
+
except ImportError:
|
|
11
|
+
SPACY_AVAILABLE = False
|
|
12
|
+
spacy = None
|
|
13
|
+
|
|
14
|
+
# Cache for loaded spaCy model
|
|
15
|
+
_nlp_cache: dict[str, Any] = {}
|
|
16
|
+
|
|
17
|
+
# Placeholder for ellipsis during spaCy processing
|
|
18
|
+
_ELLIPSIS_PLACEHOLDER = "\u2026" # Unicode ellipsis character
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _protect_ellipsis(text: str) -> str:
|
|
22
|
+
"""
|
|
23
|
+
Replace ellipsis patterns with a placeholder to prevent sentence splitting.
|
|
24
|
+
|
|
25
|
+
Handles:
|
|
26
|
+
- Spaced ellipsis: . . .
|
|
27
|
+
- Regular ellipsis: ...
|
|
28
|
+
- Unicode ellipsis: ...
|
|
29
|
+
"""
|
|
30
|
+
# Replace spaced ellipsis first (. . .)
|
|
31
|
+
text = re.sub(r"\.\s+\.\s+\.", _ELLIPSIS_PLACEHOLDER, text)
|
|
32
|
+
# Replace regular ellipsis (...)
|
|
33
|
+
text = re.sub(r"\.{3,}", _ELLIPSIS_PLACEHOLDER, text)
|
|
34
|
+
return text
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _restore_ellipsis(text: str) -> str:
|
|
38
|
+
"""Restore ellipsis placeholder back to spaced ellipsis."""
|
|
39
|
+
return text.replace(_ELLIPSIS_PLACEHOLDER, ". . .")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _get_nlp(language_model: str = "en_core_web_sm") -> Any:
|
|
43
|
+
"""Get or load a spaCy model (cached)."""
|
|
44
|
+
if not SPACY_AVAILABLE:
|
|
45
|
+
raise ImportError(
|
|
46
|
+
"spaCy is required for this feature. Install with: pip install phrasplit"
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
if language_model not in _nlp_cache:
|
|
50
|
+
try:
|
|
51
|
+
# spacy is guaranteed to be not None here due to SPACY_AVAILABLE check above
|
|
52
|
+
assert spacy is not None
|
|
53
|
+
_nlp_cache[language_model] = spacy.load(language_model)
|
|
54
|
+
except OSError:
|
|
55
|
+
raise OSError(
|
|
56
|
+
f"spaCy language model '{language_model}' not found. "
|
|
57
|
+
f"Download with: python -m spacy download {language_model}"
|
|
58
|
+
) from None
|
|
59
|
+
|
|
60
|
+
return _nlp_cache[language_model]
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def split_paragraphs(text: str) -> list[str]:
|
|
64
|
+
"""
|
|
65
|
+
Split text into paragraphs (separated by double newlines).
|
|
66
|
+
|
|
67
|
+
Args:
|
|
68
|
+
text: Input text
|
|
69
|
+
|
|
70
|
+
Returns:
|
|
71
|
+
List of paragraphs (non-empty, stripped)
|
|
72
|
+
"""
|
|
73
|
+
paragraphs = re.split(r"\n\s*\n", text)
|
|
74
|
+
return [p.strip() for p in paragraphs if p.strip()]
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def split_sentences(
|
|
78
|
+
text: str,
|
|
79
|
+
language_model: str = "en_core_web_sm",
|
|
80
|
+
) -> list[str]:
|
|
81
|
+
"""
|
|
82
|
+
Split text into sentences using spaCy.
|
|
83
|
+
|
|
84
|
+
Args:
|
|
85
|
+
text: Input text
|
|
86
|
+
language_model: spaCy language model to use
|
|
87
|
+
|
|
88
|
+
Returns:
|
|
89
|
+
List of sentences
|
|
90
|
+
"""
|
|
91
|
+
nlp = _get_nlp(language_model)
|
|
92
|
+
paragraphs = split_paragraphs(text)
|
|
93
|
+
|
|
94
|
+
if not paragraphs:
|
|
95
|
+
return []
|
|
96
|
+
|
|
97
|
+
result: list[str] = []
|
|
98
|
+
for para in paragraphs:
|
|
99
|
+
# Protect ellipsis from being treated as sentence boundaries
|
|
100
|
+
para = _protect_ellipsis(para)
|
|
101
|
+
|
|
102
|
+
# Process paragraph into sentences
|
|
103
|
+
doc = nlp(para)
|
|
104
|
+
sentences = [sent.text.strip() for sent in doc.sents if sent.text.strip()]
|
|
105
|
+
|
|
106
|
+
for sent in sentences:
|
|
107
|
+
# Restore ellipsis in the sentence
|
|
108
|
+
sent = _restore_ellipsis(sent)
|
|
109
|
+
result.append(sent)
|
|
110
|
+
|
|
111
|
+
return result
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _split_sentence_into_clauses(sentence: str) -> list[str]:
|
|
115
|
+
"""
|
|
116
|
+
Split a sentence into comma-separated parts for audiobook creation.
|
|
117
|
+
|
|
118
|
+
Splits only at commas, keeping the comma at the end of each part.
|
|
119
|
+
This creates natural pause points for text-to-speech processing.
|
|
120
|
+
|
|
121
|
+
Args:
|
|
122
|
+
sentence: A single sentence
|
|
123
|
+
|
|
124
|
+
Returns:
|
|
125
|
+
List of comma-separated parts
|
|
126
|
+
"""
|
|
127
|
+
# Pattern to split after comma followed by space
|
|
128
|
+
# Using positive lookbehind to keep comma at end of clause
|
|
129
|
+
parts = re.split(r"(?<=,)\s+", sentence)
|
|
130
|
+
|
|
131
|
+
# Filter empty parts and strip whitespace
|
|
132
|
+
clauses = [p.strip() for p in parts if p.strip()]
|
|
133
|
+
|
|
134
|
+
return clauses if clauses else [sentence]
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def split_clauses(
|
|
138
|
+
text: str,
|
|
139
|
+
language_model: str = "en_core_web_sm",
|
|
140
|
+
) -> list[str]:
|
|
141
|
+
"""
|
|
142
|
+
Split text into comma-separated parts for audiobook creation.
|
|
143
|
+
|
|
144
|
+
Uses spaCy for sentence detection, then splits each sentence at commas.
|
|
145
|
+
The comma stays at the end of each part, creating natural pause points
|
|
146
|
+
for text-to-speech processing.
|
|
147
|
+
|
|
148
|
+
Args:
|
|
149
|
+
text: Input text
|
|
150
|
+
language_model: spaCy language model to use
|
|
151
|
+
|
|
152
|
+
Returns:
|
|
153
|
+
List of comma-separated parts
|
|
154
|
+
|
|
155
|
+
Example:
|
|
156
|
+
Input: "I do like coffee, and I like wine."
|
|
157
|
+
Output: ["I do like coffee,", "and I like wine."]
|
|
158
|
+
"""
|
|
159
|
+
nlp = _get_nlp(language_model)
|
|
160
|
+
paragraphs = split_paragraphs(text)
|
|
161
|
+
|
|
162
|
+
if not paragraphs:
|
|
163
|
+
return []
|
|
164
|
+
|
|
165
|
+
result: list[str] = []
|
|
166
|
+
for para in paragraphs:
|
|
167
|
+
# Protect ellipsis from being treated as sentence boundaries
|
|
168
|
+
para = _protect_ellipsis(para)
|
|
169
|
+
|
|
170
|
+
# Process paragraph into sentences
|
|
171
|
+
doc = nlp(para)
|
|
172
|
+
sentences = [sent.text.strip() for sent in doc.sents if sent.text.strip()]
|
|
173
|
+
|
|
174
|
+
# Process each sentence into clauses
|
|
175
|
+
for sent in sentences:
|
|
176
|
+
# Restore ellipsis in the sentence
|
|
177
|
+
sent = _restore_ellipsis(sent)
|
|
178
|
+
|
|
179
|
+
# Split sentence at clause boundaries
|
|
180
|
+
clauses = _split_sentence_into_clauses(sent)
|
|
181
|
+
result.extend(clauses)
|
|
182
|
+
|
|
183
|
+
return result
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _split_at_clauses(text: str, max_length: int) -> list[str]:
|
|
187
|
+
"""
|
|
188
|
+
Split text at comma boundaries for audiobook creation.
|
|
189
|
+
|
|
190
|
+
Args:
|
|
191
|
+
text: Text to split
|
|
192
|
+
max_length: Maximum line length
|
|
193
|
+
|
|
194
|
+
Returns:
|
|
195
|
+
List of lines
|
|
196
|
+
"""
|
|
197
|
+
# Split at commas, keeping the comma with the preceding text
|
|
198
|
+
parts = re.split(r"(?<=,)\s+", text)
|
|
199
|
+
|
|
200
|
+
result: list[str] = []
|
|
201
|
+
current_line = ""
|
|
202
|
+
|
|
203
|
+
for part in parts:
|
|
204
|
+
part = part.strip()
|
|
205
|
+
if not part:
|
|
206
|
+
continue
|
|
207
|
+
|
|
208
|
+
if not current_line:
|
|
209
|
+
current_line = part
|
|
210
|
+
elif len(current_line) + 1 + len(part) <= max_length:
|
|
211
|
+
current_line += " " + part
|
|
212
|
+
else:
|
|
213
|
+
if current_line:
|
|
214
|
+
result.append(current_line)
|
|
215
|
+
current_line = part
|
|
216
|
+
|
|
217
|
+
if current_line:
|
|
218
|
+
result.append(current_line)
|
|
219
|
+
|
|
220
|
+
# If still too long, do hard split at word boundaries
|
|
221
|
+
final_result: list[str] = []
|
|
222
|
+
for line in result:
|
|
223
|
+
if len(line) > max_length:
|
|
224
|
+
final_result.extend(_hard_split(line, max_length))
|
|
225
|
+
else:
|
|
226
|
+
final_result.append(line)
|
|
227
|
+
|
|
228
|
+
return final_result if final_result else [text]
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def _hard_split(text: str, max_length: int) -> list[str]:
|
|
232
|
+
"""
|
|
233
|
+
Hard split text at word boundaries when clause splitting isn't enough.
|
|
234
|
+
|
|
235
|
+
Args:
|
|
236
|
+
text: Text to split
|
|
237
|
+
max_length: Maximum line length
|
|
238
|
+
|
|
239
|
+
Returns:
|
|
240
|
+
List of lines
|
|
241
|
+
"""
|
|
242
|
+
words = text.split()
|
|
243
|
+
result: list[str] = []
|
|
244
|
+
current_line = ""
|
|
245
|
+
|
|
246
|
+
for word in words:
|
|
247
|
+
if not current_line:
|
|
248
|
+
current_line = word
|
|
249
|
+
elif len(current_line) + 1 + len(word) <= max_length:
|
|
250
|
+
current_line += " " + word
|
|
251
|
+
else:
|
|
252
|
+
result.append(current_line)
|
|
253
|
+
current_line = word
|
|
254
|
+
|
|
255
|
+
if current_line:
|
|
256
|
+
result.append(current_line)
|
|
257
|
+
|
|
258
|
+
return result if result else [text]
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def _split_at_boundaries(text: str, max_length: int, nlp: Any) -> list[str]:
|
|
262
|
+
"""
|
|
263
|
+
Split text at sentence/clause boundaries to fit within max_length.
|
|
264
|
+
|
|
265
|
+
Args:
|
|
266
|
+
text: Text to split
|
|
267
|
+
max_length: Maximum line length
|
|
268
|
+
nlp: spaCy language model
|
|
269
|
+
|
|
270
|
+
Returns:
|
|
271
|
+
List of lines
|
|
272
|
+
"""
|
|
273
|
+
# Protect ellipsis before spaCy processing
|
|
274
|
+
protected_text = _protect_ellipsis(text)
|
|
275
|
+
|
|
276
|
+
# First, try splitting by sentences
|
|
277
|
+
doc = nlp(protected_text)
|
|
278
|
+
sentences = [sent.text.strip() for sent in doc.sents if sent.text.strip()]
|
|
279
|
+
|
|
280
|
+
result: list[str] = []
|
|
281
|
+
current_line = ""
|
|
282
|
+
|
|
283
|
+
for sent in sentences:
|
|
284
|
+
# Restore ellipsis in the sentence
|
|
285
|
+
sent = _restore_ellipsis(sent)
|
|
286
|
+
# If sentence itself exceeds max_length, split at clauses
|
|
287
|
+
if len(sent) > max_length:
|
|
288
|
+
# Flush current line first
|
|
289
|
+
if current_line:
|
|
290
|
+
result.append(current_line)
|
|
291
|
+
current_line = ""
|
|
292
|
+
# Split sentence at clause boundaries
|
|
293
|
+
clause_lines = _split_at_clauses(sent, max_length)
|
|
294
|
+
result.extend(clause_lines)
|
|
295
|
+
elif not current_line:
|
|
296
|
+
current_line = sent
|
|
297
|
+
elif len(current_line) + 1 + len(sent) <= max_length:
|
|
298
|
+
current_line += " " + sent
|
|
299
|
+
else:
|
|
300
|
+
result.append(current_line)
|
|
301
|
+
current_line = sent
|
|
302
|
+
|
|
303
|
+
if current_line:
|
|
304
|
+
result.append(current_line)
|
|
305
|
+
|
|
306
|
+
return result if result else [text]
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def split_long_lines(
|
|
310
|
+
text: str,
|
|
311
|
+
max_length: int,
|
|
312
|
+
language_model: str = "en_core_web_sm",
|
|
313
|
+
) -> list[str]:
|
|
314
|
+
"""
|
|
315
|
+
Split lines exceeding max_length at clause/sentence boundaries.
|
|
316
|
+
|
|
317
|
+
Strategy:
|
|
318
|
+
1. First try to split at sentence boundaries
|
|
319
|
+
2. If still too long, split at clause boundaries (commas, semicolons, etc.)
|
|
320
|
+
3. If still too long, split at word boundaries
|
|
321
|
+
|
|
322
|
+
Args:
|
|
323
|
+
text: Input text
|
|
324
|
+
max_length: Maximum line length in characters
|
|
325
|
+
language_model: spaCy language model to use
|
|
326
|
+
|
|
327
|
+
Returns:
|
|
328
|
+
List of lines, each within max_length
|
|
329
|
+
"""
|
|
330
|
+
nlp = _get_nlp(language_model)
|
|
331
|
+
|
|
332
|
+
lines = text.split("\n")
|
|
333
|
+
result: list[str] = []
|
|
334
|
+
|
|
335
|
+
for line in lines:
|
|
336
|
+
# Check if line is within limit
|
|
337
|
+
if len(line) <= max_length:
|
|
338
|
+
result.append(line)
|
|
339
|
+
continue
|
|
340
|
+
|
|
341
|
+
# Split the long line
|
|
342
|
+
split_lines = _split_at_boundaries(line, max_length, nlp)
|
|
343
|
+
result.extend(split_lines)
|
|
344
|
+
|
|
345
|
+
return result
|
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: phrasplit
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A simple tool to split text.
|
|
5
|
+
Author-email: Holger Nahrstaedt <nahrstaedt@gmail.com>
|
|
6
|
+
License: MIT License
|
|
7
|
+
|
|
8
|
+
Copyright (c) 2025 Holger Nahrstaedt
|
|
9
|
+
|
|
10
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
11
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
12
|
+
in the Software without restriction, including without limitation the rights
|
|
13
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
14
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
15
|
+
furnished to do so, subject to the following conditions:
|
|
16
|
+
|
|
17
|
+
The above copyright notice and this permission notice shall be included in all
|
|
18
|
+
copies or substantial portions of the Software.
|
|
19
|
+
|
|
20
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
21
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
22
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
23
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
24
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
25
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
|
+
SOFTWARE.
|
|
27
|
+
|
|
28
|
+
Project-URL: Homepage, https://github.com/holgern/phrasplit
|
|
29
|
+
Classifier: Intended Audience :: Developers
|
|
30
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
31
|
+
Classifier: Operating System :: OS Independent
|
|
32
|
+
Classifier: Programming Language :: Python :: 3
|
|
33
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
34
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
35
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
36
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
37
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
38
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
39
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
40
|
+
Requires-Python: >=3.9
|
|
41
|
+
Description-Content-Type: text/markdown
|
|
42
|
+
License-File: LICENSE
|
|
43
|
+
Requires-Dist: click>=8.0.0
|
|
44
|
+
Requires-Dist: rich>=13.0.0
|
|
45
|
+
Requires-Dist: spacy>=3.5.0
|
|
46
|
+
Provides-Extra: dev
|
|
47
|
+
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
48
|
+
Requires-Dist: ruff>=0.1.0; extra == "dev"
|
|
49
|
+
Dynamic: license-file
|
|
50
|
+
|
|
51
|
+
# phrasplit
|
|
52
|
+
|
|
53
|
+
A Python library for splitting text into sentences, clauses, or paragraphs using spaCy
|
|
54
|
+
NLP. Designed for audiobook creation and text-to-speech processing.
|
|
55
|
+
|
|
56
|
+
## Features
|
|
57
|
+
|
|
58
|
+
- **Sentence splitting**: Intelligent sentence boundary detection using spaCy
|
|
59
|
+
- **Clause splitting**: Split sentences at commas for natural pause points
|
|
60
|
+
- **Paragraph splitting**: Split text at double newlines
|
|
61
|
+
- **Long line splitting**: Break long lines at sentence/clause boundaries
|
|
62
|
+
- **Abbreviation handling**: Correctly handles Mr., Dr., U.S.A., etc.
|
|
63
|
+
- **Ellipsis support**: Preserves ellipses without incorrect splitting
|
|
64
|
+
|
|
65
|
+
## Installation
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
pip install phrasplit
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
You'll also need to download a spaCy language model:
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
python -m spacy download en_core_web_sm
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
## Quick Start
|
|
78
|
+
|
|
79
|
+
### Python API
|
|
80
|
+
|
|
81
|
+
```python
|
|
82
|
+
from phrasplit import split_sentences, split_clauses, split_paragraphs, split_long_lines
|
|
83
|
+
|
|
84
|
+
# Split text into sentences
|
|
85
|
+
text = "Dr. Smith is here. She has a Ph.D. in Chemistry."
|
|
86
|
+
sentences = split_sentences(text)
|
|
87
|
+
# ['Dr. Smith is here.', 'She has a Ph.D. in Chemistry.']
|
|
88
|
+
|
|
89
|
+
# Split sentences into comma-separated parts (for audiobook pauses)
|
|
90
|
+
text = "I like coffee, and I like tea."
|
|
91
|
+
clauses = split_clauses(text)
|
|
92
|
+
# ['I like coffee,', 'and I like tea.']
|
|
93
|
+
|
|
94
|
+
# Split text into paragraphs
|
|
95
|
+
text = "First paragraph.\n\nSecond paragraph."
|
|
96
|
+
paragraphs = split_paragraphs(text)
|
|
97
|
+
# ['First paragraph.', 'Second paragraph.']
|
|
98
|
+
|
|
99
|
+
# Split long lines at natural boundaries
|
|
100
|
+
text = "This is a very long sentence that needs to be split."
|
|
101
|
+
lines = split_long_lines(text, max_length=30)
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
### Command Line Interface
|
|
105
|
+
|
|
106
|
+
```bash
|
|
107
|
+
# Split into sentences
|
|
108
|
+
phrasplit sentences input.txt -o output.txt
|
|
109
|
+
|
|
110
|
+
# Split into clauses
|
|
111
|
+
phrasplit clauses input.txt -o output.txt
|
|
112
|
+
|
|
113
|
+
# Split into paragraphs
|
|
114
|
+
phrasplit paragraphs input.txt -o output.txt
|
|
115
|
+
|
|
116
|
+
# Split long lines (default max 80 characters)
|
|
117
|
+
phrasplit longlines input.txt -o output.txt --max-length 60
|
|
118
|
+
|
|
119
|
+
# Use a different spaCy model
|
|
120
|
+
phrasplit sentences input.txt --model en_core_web_lg
|
|
121
|
+
|
|
122
|
+
# Read from stdin (pipe or redirect)
|
|
123
|
+
echo "Hello world. This is a test." | phrasplit sentences
|
|
124
|
+
cat input.txt | phrasplit clauses -o output.txt
|
|
125
|
+
|
|
126
|
+
# Explicit stdin with dash
|
|
127
|
+
phrasplit sentences - < input.txt
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
## API Reference
|
|
131
|
+
|
|
132
|
+
### `split_sentences(text, language_model="en_core_web_sm")`
|
|
133
|
+
|
|
134
|
+
Split text into sentences using spaCy's sentence boundary detection.
|
|
135
|
+
|
|
136
|
+
**Parameters:**
|
|
137
|
+
|
|
138
|
+
- `text`: Input text string
|
|
139
|
+
- `language_model`: spaCy model to use (default: "en_core_web_sm")
|
|
140
|
+
|
|
141
|
+
**Returns:** List of sentences
|
|
142
|
+
|
|
143
|
+
### `split_clauses(text, language_model="en_core_web_sm")`
|
|
144
|
+
|
|
145
|
+
Split text into comma-separated parts. Useful for creating natural pause points in
|
|
146
|
+
audiobook/TTS applications.
|
|
147
|
+
|
|
148
|
+
**Parameters:**
|
|
149
|
+
|
|
150
|
+
- `text`: Input text string
|
|
151
|
+
- `language_model`: spaCy model to use (default: "en_core_web_sm")
|
|
152
|
+
|
|
153
|
+
**Returns:** List of clauses (comma stays at end of each part)
|
|
154
|
+
|
|
155
|
+
### `split_paragraphs(text)`
|
|
156
|
+
|
|
157
|
+
Split text into paragraphs at double newlines.
|
|
158
|
+
|
|
159
|
+
**Parameters:**
|
|
160
|
+
|
|
161
|
+
- `text`: Input text string
|
|
162
|
+
|
|
163
|
+
**Returns:** List of paragraphs
|
|
164
|
+
|
|
165
|
+
### `split_long_lines(text, max_length, language_model="en_core_web_sm")`
|
|
166
|
+
|
|
167
|
+
Split lines exceeding max_length at sentence/clause boundaries.
|
|
168
|
+
|
|
169
|
+
**Parameters:**
|
|
170
|
+
|
|
171
|
+
- `text`: Input text string
|
|
172
|
+
- `max_length`: Maximum line length in characters
|
|
173
|
+
- `language_model`: spaCy model to use (default: "en_core_web_sm")
|
|
174
|
+
|
|
175
|
+
**Returns:** List of lines, each within max_length
|
|
176
|
+
|
|
177
|
+
## Use Cases
|
|
178
|
+
|
|
179
|
+
### Audiobook Creation
|
|
180
|
+
|
|
181
|
+
Split text at commas to create natural pause points for text-to-speech:
|
|
182
|
+
|
|
183
|
+
```python
|
|
184
|
+
from phrasplit import split_clauses
|
|
185
|
+
|
|
186
|
+
text = "When the sun rose, the birds began to sing, and the day started."
|
|
187
|
+
parts = split_clauses(text)
|
|
188
|
+
# ['When the sun rose,', 'the birds began to sing,', 'and the day started.']
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
### Subtitle Generation
|
|
192
|
+
|
|
193
|
+
Split long lines to fit subtitle constraints:
|
|
194
|
+
|
|
195
|
+
```python
|
|
196
|
+
from phrasplit import split_long_lines
|
|
197
|
+
|
|
198
|
+
text = "This is a very long sentence that would not fit on a single subtitle line."
|
|
199
|
+
lines = split_long_lines(text, max_length=42)
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
### Text Processing Pipelines
|
|
203
|
+
|
|
204
|
+
```python
|
|
205
|
+
from phrasplit import split_paragraphs, split_sentences
|
|
206
|
+
|
|
207
|
+
text = open("book.txt").read()
|
|
208
|
+
|
|
209
|
+
for paragraph in split_paragraphs(text):
|
|
210
|
+
for sentence in split_sentences(paragraph):
|
|
211
|
+
process(sentence)
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
## Requirements
|
|
215
|
+
|
|
216
|
+
- Python 3.9+
|
|
217
|
+
- spaCy 3.5+
|
|
218
|
+
- click 8.0+
|
|
219
|
+
- rich 13.0+
|
|
220
|
+
|
|
221
|
+
## License
|
|
222
|
+
|
|
223
|
+
MIT License - see [LICENSE](LICENSE) for details.
|
|
224
|
+
|
|
225
|
+
## Contributing
|
|
226
|
+
|
|
227
|
+
Contributions are welcome! Please feel free to submit a Pull Request.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
phrasplit/__init__.py,sha256=ocaPp8shbH4bvPx0ZPhNjbnXCH2eA2NW1-O-10O7yYk,407
|
|
2
|
+
phrasplit/_version.py,sha256=5jwwVncvCiTnhOedfkzzxmxsggwmTBORdFL_4wq0ZeY,704
|
|
3
|
+
phrasplit/cli.py,sha256=fVKYIXI1iJlNFI7ly1VT9doOuBw_MtpYnzdFu1WZBMY,5216
|
|
4
|
+
phrasplit/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
|
+
phrasplit/splitter.py,sha256=hL7O3ilvD_vX8H3KV2gzTVtepsfc6Mc36-_LQepq5wg,9486
|
|
6
|
+
phrasplit-0.1.0.dist-info/licenses/LICENSE,sha256=9csb1sDNn0HdUPKgOTUwtb4CkvYPcFXHnkxKCS99EWQ,1074
|
|
7
|
+
phrasplit-0.1.0.dist-info/METADATA,sha256=bvfxbzm0ioVUCoOFvt1LnXPhV8Gyoi34Ue5ovVXFLxE,6742
|
|
8
|
+
phrasplit-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
9
|
+
phrasplit-0.1.0.dist-info/entry_points.txt,sha256=L029Dr_u0KwKlTib-F3EvzdcFL-1Dxr5cSUlJueQtiI,49
|
|
10
|
+
phrasplit-0.1.0.dist-info/top_level.txt,sha256=HRa3LXhhqaPNbinioT8oFCw8EFKQMkwrM4Y-6-hGpOE,10
|
|
11
|
+
phrasplit-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Holger Nahrstaedt
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
phrasplit
|