SqueakyCleanText 0.2.5__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- SqueakyCleanText-0.3.0/PKG-INFO +161 -0
- SqueakyCleanText-0.3.0/README.md +139 -0
- SqueakyCleanText-0.3.0/SqueakyCleanText.egg-info/PKG-INFO +161 -0
- {SqueakyCleanText-0.2.5 → SqueakyCleanText-0.3.0}/SqueakyCleanText.egg-info/requires.txt +1 -0
- SqueakyCleanText-0.3.0/sct/config.py +57 -0
- SqueakyCleanText-0.3.0/sct/sct.py +167 -0
- {SqueakyCleanText-0.2.5 → SqueakyCleanText-0.3.0}/sct/utils/constants.py +9 -2
- SqueakyCleanText-0.3.0/sct/utils/contact.py +50 -0
- SqueakyCleanText-0.3.0/sct/utils/datetime.py +15 -0
- SqueakyCleanText-0.3.0/sct/utils/ner.py +310 -0
- SqueakyCleanText-0.3.0/sct/utils/normtext.py +83 -0
- SqueakyCleanText-0.3.0/sct/utils/special.py +50 -0
- SqueakyCleanText-0.3.0/sct/utils/stopwords.py +35 -0
- {SqueakyCleanText-0.2.5 → SqueakyCleanText-0.3.0}/setup.py +3 -4
- SqueakyCleanText-0.3.0/tests/test_sct.py +565 -0
- SqueakyCleanText-0.2.5/PKG-INFO +0 -231
- SqueakyCleanText-0.2.5/README.md +0 -207
- SqueakyCleanText-0.2.5/SqueakyCleanText.egg-info/PKG-INFO +0 -231
- SqueakyCleanText-0.2.5/sct/config.py +0 -100
- SqueakyCleanText-0.2.5/sct/sct.py +0 -341
- SqueakyCleanText-0.2.5/sct/utils/contact.py +0 -88
- SqueakyCleanText-0.2.5/sct/utils/datetime.py +0 -26
- SqueakyCleanText-0.2.5/sct/utils/ner.py +0 -367
- SqueakyCleanText-0.2.5/sct/utils/normtext.py +0 -148
- SqueakyCleanText-0.2.5/sct/utils/special.py +0 -113
- SqueakyCleanText-0.2.5/sct/utils/stopwords.py +0 -62
- SqueakyCleanText-0.2.5/tests/test_sct.py +0 -84
- {SqueakyCleanText-0.2.5 → SqueakyCleanText-0.3.0}/LICENSE +0 -0
- {SqueakyCleanText-0.2.5 → SqueakyCleanText-0.3.0}/MANIFEST.in +0 -0
- {SqueakyCleanText-0.2.5 → SqueakyCleanText-0.3.0}/SqueakyCleanText.egg-info/SOURCES.txt +0 -0
- {SqueakyCleanText-0.2.5 → SqueakyCleanText-0.3.0}/SqueakyCleanText.egg-info/dependency_links.txt +0 -0
- {SqueakyCleanText-0.2.5 → SqueakyCleanText-0.3.0}/SqueakyCleanText.egg-info/entry_points.txt +0 -0
- {SqueakyCleanText-0.2.5 → SqueakyCleanText-0.3.0}/SqueakyCleanText.egg-info/top_level.txt +0 -0
- {SqueakyCleanText-0.2.5 → SqueakyCleanText-0.3.0}/sct/__init__.py +0 -0
- {SqueakyCleanText-0.2.5 → SqueakyCleanText-0.3.0}/sct/scripts/__init__.py +0 -0
- {SqueakyCleanText-0.2.5 → SqueakyCleanText-0.3.0}/sct/scripts/download_nltk_stopwords.py +0 -0
- {SqueakyCleanText-0.2.5 → SqueakyCleanText-0.3.0}/sct/utils/__init__.py +0 -0
- {SqueakyCleanText-0.2.5 → SqueakyCleanText-0.3.0}/sct/utils/resources.py +0 -0
- {SqueakyCleanText-0.2.5 → SqueakyCleanText-0.3.0}/setup.cfg +0 -0
- {SqueakyCleanText-0.2.5 → SqueakyCleanText-0.3.0}/tests/__init__.py +0 -0
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: SqueakyCleanText
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: A comprehensive text cleaning and preprocessing pipeline.
|
|
5
|
+
Home-page: https://github.com/rhnfzl/SqueakyCleanText
|
|
6
|
+
Author: Rehan Fazal
|
|
7
|
+
License: MIT
|
|
8
|
+
Keywords: text cleaning,text preprocessing,NLP,natural language processing
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
16
|
+
Classifier: Topic :: Text Processing
|
|
17
|
+
Requires-Python: >=3.10
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
Provides-Extra: dev
|
|
20
|
+
Provides-Extra: test
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
|
|
23
|
+
<div align="center">
|
|
24
|
+
|
|
25
|
+
# SqueakyCleanText
|
|
26
|
+
|
|
27
|
+
[](https://pypi.org/project/squeakycleantext/)
|
|
28
|
+
[](https://pypistats.org/packages/squeakycleantext)
|
|
29
|
+
[](https://github.com/rhnfzl/SqueakyCleanText/actions/workflows/python-package.yml)
|
|
30
|
+
[](https://pypi.org/project/squeakycleantext/)
|
|
31
|
+
[](LICENSE)
|
|
32
|
+
|
|
33
|
+
A comprehensive text cleaning and preprocessing pipeline for machine learning and NLP tasks.
|
|
34
|
+
</div>
|
|
35
|
+
|
|
36
|
+
In the world of machine learning and natural language processing, clean and well-structured text data is crucial for building effective downstream models and managing token limits in language models.
|
|
37
|
+
|
|
38
|
+
SqueakyCleanText simplifies the process by automatically addressing common text issues, ensuring your data is clean and well-structured with minimal effort on your part.
|
|
39
|
+
|
|
40
|
+
### Key Features
|
|
41
|
+
- **Encoding Issues**: Corrects text encoding problems and handles bad Unicode characters.
|
|
42
|
+
- **HTML and URLs**: Removes or replaces HTML tags and URLs with configurable tokens.
|
|
43
|
+
- **Contact Information**: Handles emails, phone numbers, and other contact details with customizable replacement tokens.
|
|
44
|
+
- **Named Entity Recognition (NER)**:
|
|
45
|
+
- Multi-language support (English, Dutch, German, Spanish)
|
|
46
|
+
- Ensemble voting technique for improved accuracy
|
|
47
|
+
- Configurable confidence thresholds
|
|
48
|
+
- Efficient batch processing
|
|
49
|
+
- Automatic text chunking for long documents
|
|
50
|
+
- GPU acceleration support
|
|
51
|
+
- **Text Normalization**:
|
|
52
|
+
- Removes isolated letters and symbols
|
|
53
|
+
- Normalizes whitespace
|
|
54
|
+
- Handles currency symbols
|
|
55
|
+
- Year detection and replacement
|
|
56
|
+
- Number standardization
|
|
57
|
+
- **Language Support**:
|
|
58
|
+
- Automatic language detection
|
|
59
|
+
- Language-specific NER models
|
|
60
|
+
- Language-aware stopword removal
|
|
61
|
+
- **Dual Output Formats**:
|
|
62
|
+
- Language Model format (preserves structure with tokens)
|
|
63
|
+
- Statistical Model format (optimized for classical ML)
|
|
64
|
+
- **Performance Optimization**:
|
|
65
|
+
- Batch processing support
|
|
66
|
+
- Configurable batch sizes
|
|
67
|
+
- Memory-efficient processing of large texts
|
|
68
|
+
- GPU memory management
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
### Benefits
|
|
73
|
+
|
|
74
|
+
#### For Language Models
|
|
75
|
+
- Maintains text structure while anonymizing sensitive information
|
|
76
|
+
- Configurable token replacements
|
|
77
|
+
- Preserves context while removing noise
|
|
78
|
+
- Handles long documents through intelligent chunking
|
|
79
|
+
|
|
80
|
+
#### For Statistical Models
|
|
81
|
+
- Removes stopwords and punctuation
|
|
82
|
+
- Case normalization
|
|
83
|
+
- Special symbol removal
|
|
84
|
+
- Optimized for classification tasks
|
|
85
|
+
|
|
86
|
+
#### Advanced NER Processing
|
|
87
|
+
- Ensemble approach reduces missed entities
|
|
88
|
+
- Language-specific models improve accuracy
|
|
89
|
+
- Confidence thresholds for precision control
|
|
90
|
+
- Efficient batch processing for large datasets
|
|
91
|
+
- Automatic handling of long documents
|
|
92
|
+
|
|
93
|
+
## Installation
|
|
94
|
+
|
|
95
|
+
```sh
|
|
96
|
+
pip install SqueakyCleanText
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
## Usage
|
|
100
|
+
|
|
101
|
+
### Basic Usage
|
|
102
|
+
```python
|
|
103
|
+
from sct import sct
|
|
104
|
+
|
|
105
|
+
# Initialize the TextCleaner
|
|
106
|
+
sx = sct.TextCleaner()
|
|
107
|
+
|
|
108
|
+
# Process single text
|
|
109
|
+
text = "Hey John Doe, email me at john.doe@example.com"
|
|
110
|
+
lm_text, stat_text, language = sx.process(text)
|
|
111
|
+
|
|
112
|
+
# Process multiple texts efficiently
|
|
113
|
+
texts = ["Text 1", "Text 2", "Text 3"]
|
|
114
|
+
results = sx.process_batch(texts, batch_size=2)
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
### Advanced Configuration
|
|
118
|
+
```python
|
|
119
|
+
from sct import sct, config
|
|
120
|
+
|
|
121
|
+
# Customize NER settings
|
|
122
|
+
config.CHECK_NER_PROCESS = True
|
|
123
|
+
config.NER_CONFIDENCE_THRESHOLD = 0.85
|
|
124
|
+
config.POSITIONAL_TAGS = ['PER', 'LOC', 'ORG']
|
|
125
|
+
|
|
126
|
+
# Customize replacement tokens
|
|
127
|
+
config.REPLACE_WITH_URL = "<URL>"
|
|
128
|
+
config.REPLACE_WITH_EMAIL = "<EMAIL>"
|
|
129
|
+
config.REPLACE_WITH_PHONE_NUMBERS = "<PHONE>"
|
|
130
|
+
|
|
131
|
+
# Set known language (skips detection)
|
|
132
|
+
config.LANGUAGE = "ENGLISH" # Options: ENGLISH, DUTCH, GERMAN, SPANISH
|
|
133
|
+
|
|
134
|
+
# Initialize with custom settings
|
|
135
|
+
sx = sct.TextCleaner()
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
## API
|
|
139
|
+
|
|
140
|
+
### `sct.TextCleaner`
|
|
141
|
+
|
|
142
|
+
#### `process(text: str) -> Tuple[str, str, str]`
|
|
143
|
+
|
|
144
|
+
Processes the input text and returns a tuple containing:
|
|
145
|
+
- Cleaned text formatted for language models.
|
|
146
|
+
- Cleaned text formatted for statistical models (stopwords removed).
|
|
147
|
+
- Detected language of the text.
|
|
148
|
+
|
|
149
|
+
## Contributing
|
|
150
|
+
|
|
151
|
+
Contributions are welcome! Please feel free to submit a Pull Request or open an issue.
|
|
152
|
+
|
|
153
|
+
## License
|
|
154
|
+
|
|
155
|
+
This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
|
|
156
|
+
|
|
157
|
+
## Acknowledgements
|
|
158
|
+
|
|
159
|
+
The package took inspirations from the following repo:
|
|
160
|
+
|
|
161
|
+
- [clean-text](https://github.com/jfilter/clean-text)
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
|
|
3
|
+
# SqueakyCleanText
|
|
4
|
+
|
|
5
|
+
[](https://pypi.org/project/squeakycleantext/)
|
|
6
|
+
[](https://pypistats.org/packages/squeakycleantext)
|
|
7
|
+
[](https://github.com/rhnfzl/SqueakyCleanText/actions/workflows/python-package.yml)
|
|
8
|
+
[](https://pypi.org/project/squeakycleantext/)
|
|
9
|
+
[](LICENSE)
|
|
10
|
+
|
|
11
|
+
A comprehensive text cleaning and preprocessing pipeline for machine learning and NLP tasks.
|
|
12
|
+
</div>
|
|
13
|
+
|
|
14
|
+
In the world of machine learning and natural language processing, clean and well-structured text data is crucial for building effective downstream models and managing token limits in language models.
|
|
15
|
+
|
|
16
|
+
SqueakyCleanText simplifies the process by automatically addressing common text issues, ensuring your data is clean and well-structured with minimal effort on your part.
|
|
17
|
+
|
|
18
|
+
### Key Features
|
|
19
|
+
- **Encoding Issues**: Corrects text encoding problems and handles bad Unicode characters.
|
|
20
|
+
- **HTML and URLs**: Removes or replaces HTML tags and URLs with configurable tokens.
|
|
21
|
+
- **Contact Information**: Handles emails, phone numbers, and other contact details with customizable replacement tokens.
|
|
22
|
+
- **Named Entity Recognition (NER)**:
|
|
23
|
+
- Multi-language support (English, Dutch, German, Spanish)
|
|
24
|
+
- Ensemble voting technique for improved accuracy
|
|
25
|
+
- Configurable confidence thresholds
|
|
26
|
+
- Efficient batch processing
|
|
27
|
+
- Automatic text chunking for long documents
|
|
28
|
+
- GPU acceleration support
|
|
29
|
+
- **Text Normalization**:
|
|
30
|
+
- Removes isolated letters and symbols
|
|
31
|
+
- Normalizes whitespace
|
|
32
|
+
- Handles currency symbols
|
|
33
|
+
- Year detection and replacement
|
|
34
|
+
- Number standardization
|
|
35
|
+
- **Language Support**:
|
|
36
|
+
- Automatic language detection
|
|
37
|
+
- Language-specific NER models
|
|
38
|
+
- Language-aware stopword removal
|
|
39
|
+
- **Dual Output Formats**:
|
|
40
|
+
- Language Model format (preserves structure with tokens)
|
|
41
|
+
- Statistical Model format (optimized for classical ML)
|
|
42
|
+
- **Performance Optimization**:
|
|
43
|
+
- Batch processing support
|
|
44
|
+
- Configurable batch sizes
|
|
45
|
+
- Memory-efficient processing of large texts
|
|
46
|
+
- GPU memory management
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
### Benefits
|
|
51
|
+
|
|
52
|
+
#### For Language Models
|
|
53
|
+
- Maintains text structure while anonymizing sensitive information
|
|
54
|
+
- Configurable token replacements
|
|
55
|
+
- Preserves context while removing noise
|
|
56
|
+
- Handles long documents through intelligent chunking
|
|
57
|
+
|
|
58
|
+
#### For Statistical Models
|
|
59
|
+
- Removes stopwords and punctuation
|
|
60
|
+
- Case normalization
|
|
61
|
+
- Special symbol removal
|
|
62
|
+
- Optimized for classification tasks
|
|
63
|
+
|
|
64
|
+
#### Advanced NER Processing
|
|
65
|
+
- Ensemble approach reduces missed entities
|
|
66
|
+
- Language-specific models improve accuracy
|
|
67
|
+
- Confidence thresholds for precision control
|
|
68
|
+
- Efficient batch processing for large datasets
|
|
69
|
+
- Automatic handling of long documents
|
|
70
|
+
|
|
71
|
+
## Installation
|
|
72
|
+
|
|
73
|
+
```sh
|
|
74
|
+
pip install SqueakyCleanText
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
## Usage
|
|
78
|
+
|
|
79
|
+
### Basic Usage
|
|
80
|
+
```python
|
|
81
|
+
from sct import sct
|
|
82
|
+
|
|
83
|
+
# Initialize the TextCleaner
|
|
84
|
+
sx = sct.TextCleaner()
|
|
85
|
+
|
|
86
|
+
# Process single text
|
|
87
|
+
text = "Hey John Doe, email me at john.doe@example.com"
|
|
88
|
+
lm_text, stat_text, language = sx.process(text)
|
|
89
|
+
|
|
90
|
+
# Process multiple texts efficiently
|
|
91
|
+
texts = ["Text 1", "Text 2", "Text 3"]
|
|
92
|
+
results = sx.process_batch(texts, batch_size=2)
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
### Advanced Configuration
|
|
96
|
+
```python
|
|
97
|
+
from sct import sct, config
|
|
98
|
+
|
|
99
|
+
# Customize NER settings
|
|
100
|
+
config.CHECK_NER_PROCESS = True
|
|
101
|
+
config.NER_CONFIDENCE_THRESHOLD = 0.85
|
|
102
|
+
config.POSITIONAL_TAGS = ['PER', 'LOC', 'ORG']
|
|
103
|
+
|
|
104
|
+
# Customize replacement tokens
|
|
105
|
+
config.REPLACE_WITH_URL = "<URL>"
|
|
106
|
+
config.REPLACE_WITH_EMAIL = "<EMAIL>"
|
|
107
|
+
config.REPLACE_WITH_PHONE_NUMBERS = "<PHONE>"
|
|
108
|
+
|
|
109
|
+
# Set known language (skips detection)
|
|
110
|
+
config.LANGUAGE = "ENGLISH" # Options: ENGLISH, DUTCH, GERMAN, SPANISH
|
|
111
|
+
|
|
112
|
+
# Initialize with custom settings
|
|
113
|
+
sx = sct.TextCleaner()
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
## API
|
|
117
|
+
|
|
118
|
+
### `sct.TextCleaner`
|
|
119
|
+
|
|
120
|
+
#### `process(text: str) -> Tuple[str, str, str]`
|
|
121
|
+
|
|
122
|
+
Processes the input text and returns a tuple containing:
|
|
123
|
+
- Cleaned text formatted for language models.
|
|
124
|
+
- Cleaned text formatted for statistical models (stopwords removed).
|
|
125
|
+
- Detected language of the text.
|
|
126
|
+
|
|
127
|
+
## Contributing
|
|
128
|
+
|
|
129
|
+
Contributions are welcome! Please feel free to submit a Pull Request or open an issue.
|
|
130
|
+
|
|
131
|
+
## License
|
|
132
|
+
|
|
133
|
+
This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
|
|
134
|
+
|
|
135
|
+
## Acknowledgements
|
|
136
|
+
|
|
137
|
+
The package took inspirations from the following repo:
|
|
138
|
+
|
|
139
|
+
- [clean-text](https://github.com/jfilter/clean-text)
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: SqueakyCleanText
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: A comprehensive text cleaning and preprocessing pipeline.
|
|
5
|
+
Home-page: https://github.com/rhnfzl/SqueakyCleanText
|
|
6
|
+
Author: Rehan Fazal
|
|
7
|
+
License: MIT
|
|
8
|
+
Keywords: text cleaning,text preprocessing,NLP,natural language processing
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
16
|
+
Classifier: Topic :: Text Processing
|
|
17
|
+
Requires-Python: >=3.10
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
Provides-Extra: dev
|
|
20
|
+
Provides-Extra: test
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
|
|
23
|
+
<div align="center">
|
|
24
|
+
|
|
25
|
+
# SqueakyCleanText
|
|
26
|
+
|
|
27
|
+
[](https://pypi.org/project/squeakycleantext/)
|
|
28
|
+
[](https://pypistats.org/packages/squeakycleantext)
|
|
29
|
+
[](https://github.com/rhnfzl/SqueakyCleanText/actions/workflows/python-package.yml)
|
|
30
|
+
[](https://pypi.org/project/squeakycleantext/)
|
|
31
|
+
[](LICENSE)
|
|
32
|
+
|
|
33
|
+
A comprehensive text cleaning and preprocessing pipeline for machine learning and NLP tasks.
|
|
34
|
+
</div>
|
|
35
|
+
|
|
36
|
+
In the world of machine learning and natural language processing, clean and well-structured text data is crucial for building effective downstream models and managing token limits in language models.
|
|
37
|
+
|
|
38
|
+
SqueakyCleanText simplifies the process by automatically addressing common text issues, ensuring your data is clean and well-structured with minimal effort on your part.
|
|
39
|
+
|
|
40
|
+
### Key Features
|
|
41
|
+
- **Encoding Issues**: Corrects text encoding problems and handles bad Unicode characters.
|
|
42
|
+
- **HTML and URLs**: Removes or replaces HTML tags and URLs with configurable tokens.
|
|
43
|
+
- **Contact Information**: Handles emails, phone numbers, and other contact details with customizable replacement tokens.
|
|
44
|
+
- **Named Entity Recognition (NER)**:
|
|
45
|
+
- Multi-language support (English, Dutch, German, Spanish)
|
|
46
|
+
- Ensemble voting technique for improved accuracy
|
|
47
|
+
- Configurable confidence thresholds
|
|
48
|
+
- Efficient batch processing
|
|
49
|
+
- Automatic text chunking for long documents
|
|
50
|
+
- GPU acceleration support
|
|
51
|
+
- **Text Normalization**:
|
|
52
|
+
- Removes isolated letters and symbols
|
|
53
|
+
- Normalizes whitespace
|
|
54
|
+
- Handles currency symbols
|
|
55
|
+
- Year detection and replacement
|
|
56
|
+
- Number standardization
|
|
57
|
+
- **Language Support**:
|
|
58
|
+
- Automatic language detection
|
|
59
|
+
- Language-specific NER models
|
|
60
|
+
- Language-aware stopword removal
|
|
61
|
+
- **Dual Output Formats**:
|
|
62
|
+
- Language Model format (preserves structure with tokens)
|
|
63
|
+
- Statistical Model format (optimized for classical ML)
|
|
64
|
+
- **Performance Optimization**:
|
|
65
|
+
- Batch processing support
|
|
66
|
+
- Configurable batch sizes
|
|
67
|
+
- Memory-efficient processing of large texts
|
|
68
|
+
- GPU memory management
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
### Benefits
|
|
73
|
+
|
|
74
|
+
#### For Language Models
|
|
75
|
+
- Maintains text structure while anonymizing sensitive information
|
|
76
|
+
- Configurable token replacements
|
|
77
|
+
- Preserves context while removing noise
|
|
78
|
+
- Handles long documents through intelligent chunking
|
|
79
|
+
|
|
80
|
+
#### For Statistical Models
|
|
81
|
+
- Removes stopwords and punctuation
|
|
82
|
+
- Case normalization
|
|
83
|
+
- Special symbol removal
|
|
84
|
+
- Optimized for classification tasks
|
|
85
|
+
|
|
86
|
+
#### Advanced NER Processing
|
|
87
|
+
- Ensemble approach reduces missed entities
|
|
88
|
+
- Language-specific models improve accuracy
|
|
89
|
+
- Confidence thresholds for precision control
|
|
90
|
+
- Efficient batch processing for large datasets
|
|
91
|
+
- Automatic handling of long documents
|
|
92
|
+
|
|
93
|
+
## Installation
|
|
94
|
+
|
|
95
|
+
```sh
|
|
96
|
+
pip install SqueakyCleanText
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
## Usage
|
|
100
|
+
|
|
101
|
+
### Basic Usage
|
|
102
|
+
```python
|
|
103
|
+
from sct import sct
|
|
104
|
+
|
|
105
|
+
# Initialize the TextCleaner
|
|
106
|
+
sx = sct.TextCleaner()
|
|
107
|
+
|
|
108
|
+
# Process single text
|
|
109
|
+
text = "Hey John Doe, email me at john.doe@example.com"
|
|
110
|
+
lm_text, stat_text, language = sx.process(text)
|
|
111
|
+
|
|
112
|
+
# Process multiple texts efficiently
|
|
113
|
+
texts = ["Text 1", "Text 2", "Text 3"]
|
|
114
|
+
results = sx.process_batch(texts, batch_size=2)
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
### Advanced Configuration
|
|
118
|
+
```python
|
|
119
|
+
from sct import sct, config
|
|
120
|
+
|
|
121
|
+
# Customize NER settings
|
|
122
|
+
config.CHECK_NER_PROCESS = True
|
|
123
|
+
config.NER_CONFIDENCE_THRESHOLD = 0.85
|
|
124
|
+
config.POSITIONAL_TAGS = ['PER', 'LOC', 'ORG']
|
|
125
|
+
|
|
126
|
+
# Customize replacement tokens
|
|
127
|
+
config.REPLACE_WITH_URL = "<URL>"
|
|
128
|
+
config.REPLACE_WITH_EMAIL = "<EMAIL>"
|
|
129
|
+
config.REPLACE_WITH_PHONE_NUMBERS = "<PHONE>"
|
|
130
|
+
|
|
131
|
+
# Set known language (skips detection)
|
|
132
|
+
config.LANGUAGE = "ENGLISH" # Options: ENGLISH, DUTCH, GERMAN, SPANISH
|
|
133
|
+
|
|
134
|
+
# Initialize with custom settings
|
|
135
|
+
sx = sct.TextCleaner()
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
## API
|
|
139
|
+
|
|
140
|
+
### `sct.TextCleaner`
|
|
141
|
+
|
|
142
|
+
#### `process(text: str) -> Tuple[str, str, str]`
|
|
143
|
+
|
|
144
|
+
Processes the input text and returns a tuple containing:
|
|
145
|
+
- Cleaned text formatted for language models.
|
|
146
|
+
- Cleaned text formatted for statistical models (stopwords removed).
|
|
147
|
+
- Detected language of the text.
|
|
148
|
+
|
|
149
|
+
## Contributing
|
|
150
|
+
|
|
151
|
+
Contributions are welcome! Please feel free to submit a Pull Request or open an issue.
|
|
152
|
+
|
|
153
|
+
## License
|
|
154
|
+
|
|
155
|
+
This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
|
|
156
|
+
|
|
157
|
+
## Acknowledgements
|
|
158
|
+
|
|
159
|
+
The package took inspirations from the following repo:
|
|
160
|
+
|
|
161
|
+
- [clean-text](https://github.com/jfilter/clean-text)
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""
|
|
2
|
+
detect_language : to detect the language automatically, but would consume more time if done on a batch
|
|
3
|
+
fix_bad_unicode : if True, fix "broken" unicode such as mojibake and garbled HTML entities
|
|
4
|
+
to_ascii_unicode : if True, convert non-to_ascii characters into their closest to_ascii equivalents
|
|
5
|
+
replace_with_url : special URL token, default "",
|
|
6
|
+
replace_with_email : special EMAIL token, default "",
|
|
7
|
+
replace_years : replace year, default "",
|
|
8
|
+
replace_with_phone_number : special PHONE token, default "",
|
|
9
|
+
replace_with_number : special NUMBER token, default "",
|
|
10
|
+
no_currency_symbols : if True, replace all currency symbols with the respective alphabetical ones,
|
|
11
|
+
ner_process : To execute NER Process to remove the positpositional tags, PER, LOC, ORG, MISC
|
|
12
|
+
remove_isolated_letters : remove any isolated letters which doesn't add any value to the text
|
|
13
|
+
remove_isolated_symbols : remove any isolated symbols which shouldn't be present in the text, usually which isn't
|
|
14
|
+
immediatly prefixed and suffixed by letter or number
|
|
15
|
+
normalize_whitespace : remove any unnecessary whitespace
|
|
16
|
+
statistical_model_processing : to get the statistical model text, like for fastText, SVM, LR etc
|
|
17
|
+
casefold : to lower the text
|
|
18
|
+
remove_stopwords : remove stopwords based on the language, usues NLTK stopwords
|
|
19
|
+
remove_punctuation : removes all the special symbols
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
CHECK_DETECT_LANGUAGE = True
|
|
23
|
+
CHECK_FIX_BAD_UNICODE = True
|
|
24
|
+
CHECK_TO_ASCII_UNICODE = True
|
|
25
|
+
CHECK_REPLACE_HTML = True
|
|
26
|
+
CHECK_REPLACE_URLS = True
|
|
27
|
+
CHECK_REPLACE_EMAILS = True
|
|
28
|
+
CHECK_REPLACE_YEARS = True
|
|
29
|
+
CHECK_REPLACE_PHONE_NUMBERS = True
|
|
30
|
+
CHECK_REPLACE_NUMBERS = True
|
|
31
|
+
CHECK_REPLACE_CURRENCY_SYMBOLS = True
|
|
32
|
+
CHECK_NER_PROCESS = True
|
|
33
|
+
CHECK_REMOVE_ISOLATED_LETTERS = True
|
|
34
|
+
CHECK_REMOVE_ISOLATED_SPECIAL_SYMBOLS = True
|
|
35
|
+
CHECK_NORMALIZE_WHITESPACE = True
|
|
36
|
+
CHECK_STATISTICAL_MODEL_PROCESSING = True
|
|
37
|
+
CHECK_CASEFOLD = True
|
|
38
|
+
CHECK_REMOVE_STOPWORDS = True
|
|
39
|
+
CHECK_REMOVE_PUNCTUATION = True
|
|
40
|
+
CHECK_REMOVE_STEXT_CUSTOM_STOP_WORDS = True
|
|
41
|
+
REPLACE_WITH_URL = "<URL>"
|
|
42
|
+
REPLACE_WITH_HTML = "<HTML>"
|
|
43
|
+
REPLACE_WITH_EMAIL = "<EMAIL>"
|
|
44
|
+
REPLACE_WITH_YEARS = "<YEAR>"
|
|
45
|
+
REPLACE_WITH_PHONE_NUMBERS = "<PHONE>"
|
|
46
|
+
REPLACE_WITH_NUMBERS = "<NUMBER>"
|
|
47
|
+
REPLACE_WITH_CURRENCY_SYMBOLS = None
|
|
48
|
+
POSITIONAL_TAGS = ['PER', 'LOC', 'ORG']
|
|
49
|
+
NER_CONFIDENCE_THRESHOLD = 0.85
|
|
50
|
+
LANGUAGE = None
|
|
51
|
+
|
|
52
|
+
# Order of the model is Important : English Model, Dutch Model, German Model, Spanish Model, MULTILINGUAL Model
|
|
53
|
+
NER_MODELS_LIST = ["FacebookAI/xlm-roberta-large-finetuned-conll03-english",
|
|
54
|
+
"FacebookAI/xlm-roberta-large-finetuned-conll02-dutch",
|
|
55
|
+
"FacebookAI/xlm-roberta-large-finetuned-conll03-german",
|
|
56
|
+
"FacebookAI/xlm-roberta-large-finetuned-conll02-spanish",
|
|
57
|
+
"Babelscape/wikineural-multilingual-ner"]
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
"""
|
|
2
|
+
This code provides a comprehensive text cleaning and preprocessing pipeline.
|
|
3
|
+
It includes functions to normalize, remove personal information and clean text data,
|
|
4
|
+
which is crucial for natural language processing tasks.
|
|
5
|
+
"""
|
|
6
|
+
from sct import config
|
|
7
|
+
from sct.utils import contact, datetime, ner, normtext, resources, special, stopwords
|
|
8
|
+
from typing import List, Any
|
|
9
|
+
|
|
10
|
+
class TextCleaner:
|
|
11
|
+
|
|
12
|
+
def __init__(self):
|
|
13
|
+
self.ProcessContacts = contact.ProcessContacts()
|
|
14
|
+
self.ProcessDateTime = datetime.ProcessDateTime()
|
|
15
|
+
self.ProcessSpecialSymbols = special.ProcessSpecialSymbols()
|
|
16
|
+
self.NormaliseText = normtext.NormaliseText()
|
|
17
|
+
self.ProcessStopwords = stopwords.ProcessStopwords()
|
|
18
|
+
self.GeneralNER = ner.GeneralNER()
|
|
19
|
+
self.pipeline = []
|
|
20
|
+
self.language = None
|
|
21
|
+
self.batch_size = 8 # Default batch size for NER
|
|
22
|
+
self.init_pipeline()
|
|
23
|
+
|
|
24
|
+
def init_pipeline(self):
|
|
25
|
+
# Initialize pipeline steps based on config
|
|
26
|
+
language_config = config.LANGUAGE.lower() if config.LANGUAGE else None
|
|
27
|
+
|
|
28
|
+
if language_config and language_config in resources.LANGUAGE_NAME:
|
|
29
|
+
self.language = language_config.upper()
|
|
30
|
+
elif any([config.CHECK_DETECT_LANGUAGE, config.CHECK_NER_PROCESS, config.CHECK_REMOVE_STOPWORDS]):
|
|
31
|
+
self.pipeline.append(self.detect_language)
|
|
32
|
+
|
|
33
|
+
if config.CHECK_FIX_BAD_UNICODE:
|
|
34
|
+
self.pipeline.append(self.fix_bad_unicode)
|
|
35
|
+
if config.CHECK_TO_ASCII_UNICODE:
|
|
36
|
+
self.pipeline.append(self.to_ascii_unicode)
|
|
37
|
+
if config.CHECK_REPLACE_HTML:
|
|
38
|
+
self.pipeline.append(self.replace_html)
|
|
39
|
+
if config.CHECK_REPLACE_URLS:
|
|
40
|
+
self.pipeline.append(self.replace_urls)
|
|
41
|
+
if config.CHECK_REPLACE_EMAILS:
|
|
42
|
+
self.pipeline.append(self.replace_emails)
|
|
43
|
+
if config.CHECK_REPLACE_YEARS:
|
|
44
|
+
self.pipeline.append(self.replace_years)
|
|
45
|
+
if config.CHECK_REPLACE_PHONE_NUMBERS:
|
|
46
|
+
self.pipeline.append(self.replace_phone_numbers)
|
|
47
|
+
if config.CHECK_REPLACE_NUMBERS:
|
|
48
|
+
self.pipeline.append(self.replace_numbers)
|
|
49
|
+
if config.CHECK_REPLACE_CURRENCY_SYMBOLS:
|
|
50
|
+
self.pipeline.append(self.replace_currency_symbols)
|
|
51
|
+
|
|
52
|
+
if config.CHECK_NER_PROCESS:
|
|
53
|
+
self.pipeline.append(self.ner_process)
|
|
54
|
+
|
|
55
|
+
if config.CHECK_REMOVE_ISOLATED_LETTERS:
|
|
56
|
+
self.pipeline.append(self.remove_isolated_letters)
|
|
57
|
+
if config.CHECK_REMOVE_ISOLATED_SPECIAL_SYMBOLS:
|
|
58
|
+
self.pipeline.append(self.remove_isolated_special_symbols)
|
|
59
|
+
if config.CHECK_NORMALIZE_WHITESPACE:
|
|
60
|
+
self.pipeline.append(self.normalize_whitespace)
|
|
61
|
+
|
|
62
|
+
def process_batch(self, texts: List[str], batch_size: int = None) -> List[Any]:
|
|
63
|
+
"""Process multiple texts efficiently in batches."""
|
|
64
|
+
if not texts:
|
|
65
|
+
return []
|
|
66
|
+
|
|
67
|
+
results = []
|
|
68
|
+
batch_size = batch_size or self.batch_size
|
|
69
|
+
|
|
70
|
+
for text in texts:
|
|
71
|
+
# Validate input type and content
|
|
72
|
+
if not isinstance(text, str):
|
|
73
|
+
raise ValueError(f"Input must be string, got {type(text)}")
|
|
74
|
+
|
|
75
|
+
# Handle empty text case
|
|
76
|
+
if not text or text.isspace():
|
|
77
|
+
results.append(("", "", None))
|
|
78
|
+
continue
|
|
79
|
+
|
|
80
|
+
current_text = text # No need for str() conversion now
|
|
81
|
+
|
|
82
|
+
# Reset language for each text
|
|
83
|
+
self.language = None
|
|
84
|
+
|
|
85
|
+
# Apply non-NER pipeline steps
|
|
86
|
+
for step in [s for s in self.pipeline if s != self.ner_process]:
|
|
87
|
+
current_text = step(current_text)
|
|
88
|
+
|
|
89
|
+
# Batch NER processing if enabled
|
|
90
|
+
if config.CHECK_NER_PROCESS:
|
|
91
|
+
current_text = self.GeneralNER.ner_process(
|
|
92
|
+
current_text,
|
|
93
|
+
positional_tags=config.POSITIONAL_TAGS,
|
|
94
|
+
ner_confidence_threshold=config.NER_CONFIDENCE_THRESHOLD,
|
|
95
|
+
language=self.language
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
# Format results
|
|
99
|
+
if config.CHECK_STATISTICAL_MODEL_PROCESSING:
|
|
100
|
+
stext = self.statistical_model_processing(current_text)
|
|
101
|
+
results.append((current_text, stext, self.language))
|
|
102
|
+
elif config.CHECK_DETECT_LANGUAGE:
|
|
103
|
+
results.append((current_text, self.language))
|
|
104
|
+
else:
|
|
105
|
+
results.append(current_text)
|
|
106
|
+
|
|
107
|
+
return results
|
|
108
|
+
|
|
109
|
+
def process(self, text: str) -> Any:
|
|
110
|
+
"""Process a single text. Maintains backward compatibility."""
|
|
111
|
+
return self.process_batch([text])[0]
|
|
112
|
+
|
|
113
|
+
def detect_language(self, text):
|
|
114
|
+
self.language = str(resources.DETECTOR.detect_language_of(text)).split(".")[-1]
|
|
115
|
+
return text
|
|
116
|
+
|
|
117
|
+
def fix_bad_unicode(self, text):
|
|
118
|
+
return self.NormaliseText.fix_bad_unicode(text)
|
|
119
|
+
|
|
120
|
+
def to_ascii_unicode(self, text):
|
|
121
|
+
return self.NormaliseText.to_ascii_unicode(text)
|
|
122
|
+
|
|
123
|
+
def replace_html(self, text):
|
|
124
|
+
return self.ProcessContacts.replace_html(text, replace_with=config.REPLACE_WITH_HTML)
|
|
125
|
+
|
|
126
|
+
def replace_urls(self, text):
|
|
127
|
+
return self.ProcessContacts.replace_urls(text, replace_with=config.REPLACE_WITH_URL)
|
|
128
|
+
|
|
129
|
+
def replace_emails(self, text):
|
|
130
|
+
return self.ProcessContacts.replace_emails(text, replace_with=config.REPLACE_WITH_EMAIL)
|
|
131
|
+
|
|
132
|
+
def replace_years(self, text):
|
|
133
|
+
return self.ProcessDateTime.replace_years(text, replace_with=config.REPLACE_WITH_YEARS)
|
|
134
|
+
|
|
135
|
+
def replace_phone_numbers(self, text):
|
|
136
|
+
return self.ProcessContacts.replace_phone_numbers(text, replace_with=config.REPLACE_WITH_PHONE_NUMBERS)
|
|
137
|
+
|
|
138
|
+
def replace_numbers(self, text):
|
|
139
|
+
return self.ProcessContacts.replace_numbers(text, replace_with=config.REPLACE_WITH_NUMBERS)
|
|
140
|
+
|
|
141
|
+
def replace_currency_symbols(self, text):
|
|
142
|
+
return self.ProcessSpecialSymbols.replace_currency_symbols(text, replace_with=config.REPLACE_WITH_CURRENCY_SYMBOLS)
|
|
143
|
+
|
|
144
|
+
def ner_process(self, text):
|
|
145
|
+
return self.GeneralNER.ner_process(text, config.POSITIONAL_TAGS, config.NER_CONFIDENCE_THRESHOLD, self.language)
|
|
146
|
+
|
|
147
|
+
def remove_isolated_letters(self, text):
|
|
148
|
+
return self.ProcessSpecialSymbols.remove_isolated_letters(text)
|
|
149
|
+
|
|
150
|
+
def remove_isolated_special_symbols(self, text):
|
|
151
|
+
return self.ProcessSpecialSymbols.remove_isolated_special_symbols(text)
|
|
152
|
+
|
|
153
|
+
def normalize_whitespace(self, text):
|
|
154
|
+
return self.NormaliseText.normalize_whitespace(text, no_line_breaks=True)
|
|
155
|
+
|
|
156
|
+
def statistical_model_processing(self, text):
|
|
157
|
+
if config.CHECK_CASEFOLD:
|
|
158
|
+
stext = text.casefold() # lowercase
|
|
159
|
+
if config.CHECK_REMOVE_STOPWORDS:
|
|
160
|
+
stext = self.ProcessStopwords.remove_stopwords(stext, self.language)
|
|
161
|
+
if config.CHECK_REMOVE_PUNCTUATION:
|
|
162
|
+
stext = self.ProcessSpecialSymbols.remove_punctuation(stext)
|
|
163
|
+
if config.CHECK_REMOVE_ISOLATED_LETTERS:
|
|
164
|
+
stext = self.ProcessSpecialSymbols.remove_isolated_letters(stext)
|
|
165
|
+
if config.CHECK_NORMALIZE_WHITESPACE:
|
|
166
|
+
stext = self.NormaliseText.normalize_whitespace(stext)
|
|
167
|
+
return stext
|