valx 0.2.6__tar.gz → 0.2.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {valx-0.2.6 → valx-0.2.7}/PKG-INFO +76 -113
- valx-0.2.6/valx.egg-info/PKG-INFO → valx-0.2.7/README.md +64 -125
- valx-0.2.7/pyproject.toml +36 -0
- {valx-0.2.6 → valx-0.2.7}/valx/__init__.py +173 -265
- valx-0.2.6/README.md → valx-0.2.7/valx.egg-info/PKG-INFO +88 -93
- {valx-0.2.6 → valx-0.2.7}/valx.egg-info/SOURCES.txt +1 -1
- {valx-0.2.6 → valx-0.2.7}/valx.egg-info/requires.txt +1 -0
- valx-0.2.6/setup.py +0 -33
- {valx-0.2.6 → valx-0.2.7}/LICENSE +0 -0
- {valx-0.2.6 → valx-0.2.7}/setup.cfg +0 -0
- {valx-0.2.6 → valx-0.2.7}/valx/models/classifier_upgraded.pkl +0 -0
- {valx-0.2.6 → valx-0.2.7}/valx/models/count_vectorizer.sav +0 -0
- {valx-0.2.6 → valx-0.2.7}/valx/models/decision_tree_model.sav +0 -0
- {valx-0.2.6 → valx-0.2.7}/valx.egg-info/dependency_links.txt +0 -0
- {valx-0.2.6 → valx-0.2.7}/valx.egg-info/top_level.txt +0 -0
|
@@ -1,53 +1,44 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: valx
|
|
3
|
-
Version: 0.2.
|
|
4
|
-
Summary:
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
3
|
+
Version: 0.2.7
|
|
4
|
+
Summary: Text filters for profanity, sensitive information, and offensive speech.
|
|
5
|
+
Author-email: Infinitode Pty Ltd <infinitode.ltd@gmail.com>
|
|
6
|
+
License-Expression: LicenseRef-ValX-Modified-MIT
|
|
7
|
+
Project-URL: Repository, https://github.com/Infinitode/ValX
|
|
8
|
+
Project-URL: Issues, https://github.com/Infinitode/ValX/issues
|
|
9
|
+
Project-URL: Documentation, https://infinitode-docs.gitbook.io/documentation/package-documentation/valx-package-documentation
|
|
8
10
|
Classifier: Development Status :: 5 - Production/Stable
|
|
9
11
|
Classifier: Intended Audience :: Developers
|
|
10
|
-
Classifier:
|
|
11
|
-
Classifier: Programming Language :: Python :: 3.6
|
|
12
|
-
Classifier: Programming Language :: Python :: 3.7
|
|
13
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
14
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
13
|
Classifier: Programming Language :: Python :: 3.10
|
|
16
14
|
Classifier: Programming Language :: Python :: 3.11
|
|
17
15
|
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
-
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
18
|
+
Requires-Python: >=3.10
|
|
19
19
|
Description-Content-Type: text/markdown
|
|
20
20
|
License-File: LICENSE
|
|
21
|
+
Requires-Dist: packaging
|
|
21
22
|
Requires-Dist: scikit-learn
|
|
22
|
-
Dynamic: author
|
|
23
|
-
Dynamic: author-email
|
|
24
|
-
Dynamic: classifier
|
|
25
|
-
Dynamic: description
|
|
26
|
-
Dynamic: description-content-type
|
|
27
|
-
Dynamic: home-page
|
|
28
23
|
Dynamic: license-file
|
|
29
|
-
Dynamic: requires-dist
|
|
30
|
-
Dynamic: requires-python
|
|
31
|
-
Dynamic: summary
|
|
32
24
|
|
|
33
25
|
# ValX
|
|
34
26
|
|
|
35
|
-

|
|
36
28
|
[](https://github.com/infinitode/valx)
|
|
37
29
|

|
|
38
30
|

|
|
39
31
|

|
|
32
|
+
[](https://pypi.org/project/valx/#security)
|
|
40
33
|
|
|
41
34
|
An open-source Python library for data cleaning tasks. It includes functions for profanity detection, and removal, and detection and removal of personal information. Also includes hate speech and offensive language detection and removal, using AI.
|
|
42
35
|
|
|
43
|
-
|
|
44
|
-
> If you are using `scikit-learn` versions older than version `1.3.0`, please also downgrade your version of `numpy` as stated below. Otherwise, you can continue to use your preferred version of `scikit-learn` without downgrading `numpy`.
|
|
45
|
-
>
|
|
46
|
-
> Please downgrade to `numpy` version `1.26.4`. Our ValX **DecisionTreeClassifier** AI model, relies on lower versions of `numpy`, because it was trained on these versions.
|
|
47
|
-
> For more information see: https://techoverflow.net/2024/07/23/how-to-fix-numpy-dtype-size-changed-may-indicate-binary-incompatibility-expected-96-from-c-header-got-88-from-pyobject/
|
|
36
|
+
## Changes in 0.2.7
|
|
48
37
|
|
|
49
|
-
|
|
50
|
-
|
|
38
|
+
- Sensitive information redaction now uses exact match positions on each line and merges overlaps, so it won't redact unrelated copies of a substring or leave part of an overlapping match exposed. Parenthesized phone area codes and signed coordinates are included in the matches.
|
|
39
|
+
- Profanity filtering supports the documented Finnish `FI` alias and punctuation in custom terms. Detections are in text order and the `Word` value preserves the original casing. Lines surrounded by `$` are no longer silently dropped.
|
|
40
|
+
- The bundled hate-speech models load on first use instead of at import time. This release requires Python 3.10 or newer.
|
|
41
|
+
- The release workflow builds and tests the wheel and source distribution before publishing both to PyPI with signed attestations.
|
|
51
42
|
|
|
52
43
|
## Changes in 0.2.6
|
|
53
44
|
|
|
@@ -117,16 +108,7 @@ pip install valx
|
|
|
117
108
|
|
|
118
109
|
## Supported Python Versions
|
|
119
110
|
|
|
120
|
-
ValX
|
|
121
|
-
|
|
122
|
-
- Python 3.6
|
|
123
|
-
- Python 3.7
|
|
124
|
-
- Python 3.8
|
|
125
|
-
- Python 3.9
|
|
126
|
-
- Python 3.10
|
|
127
|
-
- Python 3.11/Later (Preferred)
|
|
128
|
-
|
|
129
|
-
Please ensure that you have one of these Python versions installed before using ValX. ValX may not work as expected on lower versions of Python than the supported.
|
|
111
|
+
ValX requires Python 3.10 or newer. CI covers Python 3.10 through 3.14.
|
|
130
112
|
|
|
131
113
|
## Features
|
|
132
114
|
|
|
@@ -139,64 +121,38 @@ Please ensure that you have one of these Python versions installed before using
|
|
|
139
121
|
|
|
140
122
|
### List of supported languages for profanity detection and removal
|
|
141
123
|
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
- Kabyle
|
|
175
|
-
- KAB
|
|
176
|
-
- Korean
|
|
177
|
-
- KO
|
|
178
|
-
- Dutch
|
|
179
|
-
- NL
|
|
180
|
-
- Norwegian
|
|
181
|
-
- NO
|
|
182
|
-
- Polish
|
|
183
|
-
- PL
|
|
184
|
-
- Portuguese
|
|
185
|
-
- PT
|
|
186
|
-
- Russian
|
|
187
|
-
- RU
|
|
188
|
-
- Spanish
|
|
189
|
-
- ES
|
|
190
|
-
- Swedish
|
|
191
|
-
- SV
|
|
192
|
-
- Thai
|
|
193
|
-
- TH
|
|
194
|
-
- Klingon
|
|
195
|
-
- TLH
|
|
196
|
-
- Turkish
|
|
197
|
-
- TR
|
|
198
|
-
- Chinese
|
|
199
|
-
- ZH
|
|
124
|
+
Pass a language name or code from the table as `language`. Both are case-insensitive. Use `All` to combine all built-in lists.
|
|
125
|
+
|
|
126
|
+
| Language name | Code |
|
|
127
|
+
| --- | --- |
|
|
128
|
+
| Arabic | `AR` |
|
|
129
|
+
| Czech | `CS` |
|
|
130
|
+
| Danish | `DA` |
|
|
131
|
+
| German | `DE` |
|
|
132
|
+
| English | `EN` |
|
|
133
|
+
| Esperanto | `EO` |
|
|
134
|
+
| Persian | `FA` |
|
|
135
|
+
| Finnish | `FI` |
|
|
136
|
+
| Filipino | `FIL` |
|
|
137
|
+
| French | `FR` |
|
|
138
|
+
| French (CA) | `FR-CA-U-SD-CAQC` |
|
|
139
|
+
| Hindi | `HI` |
|
|
140
|
+
| Hungarian | `HU` |
|
|
141
|
+
| Italian | `IT` |
|
|
142
|
+
| Japanese | `JA` |
|
|
143
|
+
| Kabyle | `KAB` |
|
|
144
|
+
| Korean | `KO` |
|
|
145
|
+
| Dutch | `NL` |
|
|
146
|
+
| Norwegian | `NO` |
|
|
147
|
+
| Polish | `PL` |
|
|
148
|
+
| Portuguese | `PT` |
|
|
149
|
+
| Russian | `RU` |
|
|
150
|
+
| Spanish | `ES` |
|
|
151
|
+
| Swedish | `SV` |
|
|
152
|
+
| Thai | `TH` |
|
|
153
|
+
| Klingon | `TLH` |
|
|
154
|
+
| Turkish | `TR` |
|
|
155
|
+
| Chinese | `ZH` |
|
|
200
156
|
|
|
201
157
|
## Usage
|
|
202
158
|
|
|
@@ -242,7 +198,7 @@ results_custom_plus_english = detect_profanity(sample_text, language="English",
|
|
|
242
198
|
# Language will be "Custom + English"
|
|
243
199
|
# [
|
|
244
200
|
# {'Line': 1, 'Column': 15, 'Word': 'custombadword1', 'Language': 'Custom + English'},
|
|
245
|
-
# {'Line': 1, 'Column':
|
|
201
|
+
# {'Line': 1, 'Column': 39, 'Word': 'asshole', 'Language': 'Custom + English'}
|
|
246
202
|
# ]
|
|
247
203
|
print(results_custom_plus_english)
|
|
248
204
|
```
|
|
@@ -285,11 +241,11 @@ print(results_file_only)
|
|
|
285
241
|
|
|
286
242
|
**Output Format for `detect_profanity`**
|
|
287
243
|
|
|
288
|
-
The `detect_profanity` function returns
|
|
244
|
+
The `detect_profanity` function returns non-overlapping matches in text order. When two terms match at the same position, the longer one wins. Each dictionary includes:
|
|
289
245
|
|
|
290
246
|
- `"Line"`: The line number (1-indexed).
|
|
291
247
|
- `"Column"`: The column number (1-indexed) where the profanity starts.
|
|
292
|
-
- `"Word"`: The
|
|
248
|
+
- `"Word"`: The exact text that matched, with its original casing.
|
|
293
249
|
- `"Language"`: Indicates the source of the word list:
|
|
294
250
|
- `<LanguageName>` (e.g., "English"): If only a built-in language list was used.
|
|
295
251
|
- `"Custom"`: If `language=None` and only a `custom_words_list` was used.
|
|
@@ -323,26 +279,23 @@ print(cleaned_text)
|
|
|
323
279
|
|
|
324
280
|
The `load_profanity_words` function (used internally) also accepts `language` and `custom_words_list` if you need direct access to the word lists.
|
|
325
281
|
|
|
326
|
-
### Detect Sensitive Information
|
|
282
|
+
### Detect and Remove Sensitive Information
|
|
327
283
|
|
|
328
284
|
```python
|
|
329
|
-
from valx import detect_sensitive_information
|
|
285
|
+
from valx import detect_sensitive_information, remove_sensitive_information
|
|
330
286
|
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
287
|
+
lines = ["Contact help@example.com or call (555) 123-4567."]
|
|
288
|
+
print(detect_sensitive_information(lines, info_type="email"))
|
|
289
|
+
# [(1, 8, 'Email', 'help@example.com')]
|
|
334
290
|
|
|
335
|
-
|
|
336
|
-
|
|
291
|
+
print(remove_sensitive_information(lines, info_type=["email", "phone"]))
|
|
292
|
+
# ['Contact [SENSITIVE] or call [SENSITIVE].']
|
|
293
|
+
```
|
|
337
294
|
|
|
338
|
-
|
|
295
|
+
Pass one type, a list of types, or omit `info_type` to scan for all types. Detection reports a one-based line number and a **zero-based** column. Removal merges overlapping matches before inserting `[SENSITIVE]`.
|
|
339
296
|
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
# Remove sensitive information
|
|
344
|
-
cleaned_text = remove_sensitive_information(sample_text2)
|
|
345
|
-
```
|
|
297
|
+
> [!IMPORTANT]
|
|
298
|
+
> These are pattern-based filters, not a guarantee of anonymization. They can miss sensitive values or flag ordinary text. For example, `file_path` looks for filenames with an extension and can miss filenames containing spaces. Review redacted text before sharing it.
|
|
346
299
|
|
|
347
300
|
### Detect Hate Speech And Offensive Language
|
|
348
301
|
|
|
@@ -363,6 +316,16 @@ outcome_of_detection = detect_hate_speech("You are stupid.")
|
|
|
363
316
|
> [!NOTE]
|
|
364
317
|
> See our [official documentation](https://infinitode-docs.gitbook.io/documentation/package-documentation/valx-package-documentation) for more examples on how to use **ValX**.
|
|
365
318
|
|
|
319
|
+
### Model compatibility
|
|
320
|
+
|
|
321
|
+
Only the hate-speech functions load the bundled scikit-learn models. Scikit-learn can warn when loading a model saved with a different version, and compatibility across versions is not guaranteed. If you use scikit-learn 1.2.2, use NumPy 1.x with it. The rule-based profanity and sensitive-information filters do not load either model.
|
|
322
|
+
|
|
323
|
+
## PyPI provenance
|
|
324
|
+
|
|
325
|
+
Releases published by the Trusted Publishing workflow attach a [PEP 740 publish attestation](https://docs.pypi.org/attestations/publish/v1/) to each distribution file. The [PyPI Security tab](https://pypi.org/project/valx/#security) shows the status and lets you inspect the wheel and source archive separately. Older uploads are not retroactively attested.
|
|
326
|
+
|
|
327
|
+
For setup and verification steps, see the [maintainer release guide](https://github.com/Infinitode/ValX/blob/main/CONTRIBUTING.md#releasing-to-pypi).
|
|
328
|
+
|
|
366
329
|
## Contributing
|
|
367
330
|
|
|
368
331
|
Contributions are welcome! If you encounter any issues, have suggestions, or want to contribute to ValX, please open an issue or submit a pull request on [GitHub](https://github.com/infinitode/valx).
|
|
@@ -1,53 +1,20 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: valx
|
|
3
|
-
Version: 0.2.6
|
|
4
|
-
Summary: An open-source Python library for data cleaning tasks. Includes profanity detection, and removal. Now includes offensive language and hate speech detection using an AI model.
|
|
5
|
-
Home-page: https://github.com/infinitode/valx
|
|
6
|
-
Author: Infinitode Pty Ltd
|
|
7
|
-
Author-email: infinitode.ltd@gmail.com
|
|
8
|
-
Classifier: Development Status :: 5 - Production/Stable
|
|
9
|
-
Classifier: Intended Audience :: Developers
|
|
10
|
-
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
-
Classifier: Programming Language :: Python :: 3.6
|
|
12
|
-
Classifier: Programming Language :: Python :: 3.7
|
|
13
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
14
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
15
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
-
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
-
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
-
Requires-Python: >=3.6
|
|
19
|
-
Description-Content-Type: text/markdown
|
|
20
|
-
License-File: LICENSE
|
|
21
|
-
Requires-Dist: scikit-learn
|
|
22
|
-
Dynamic: author
|
|
23
|
-
Dynamic: author-email
|
|
24
|
-
Dynamic: classifier
|
|
25
|
-
Dynamic: description
|
|
26
|
-
Dynamic: description-content-type
|
|
27
|
-
Dynamic: home-page
|
|
28
|
-
Dynamic: license-file
|
|
29
|
-
Dynamic: requires-dist
|
|
30
|
-
Dynamic: requires-python
|
|
31
|
-
Dynamic: summary
|
|
32
|
-
|
|
33
1
|
# ValX
|
|
34
2
|
|
|
35
|
-

|
|
36
4
|
[](https://github.com/infinitode/valx)
|
|
37
5
|

|
|
38
6
|

|
|
39
7
|

|
|
8
|
+
[](https://pypi.org/project/valx/#security)
|
|
40
9
|
|
|
41
10
|
An open-source Python library for data cleaning tasks. It includes functions for profanity detection, and removal, and detection and removal of personal information. Also includes hate speech and offensive language detection and removal, using AI.
|
|
42
11
|
|
|
43
|
-
|
|
44
|
-
> If you are using `scikit-learn` versions older than version `1.3.0`, please also downgrade your version of `numpy` as stated below. Otherwise, you can continue to use your preferred version of `scikit-learn` without downgrading `numpy`.
|
|
45
|
-
>
|
|
46
|
-
> Please downgrade to `numpy` version `1.26.4`. Our ValX **DecisionTreeClassifier** AI model, relies on lower versions of `numpy`, because it was trained on these versions.
|
|
47
|
-
> For more information see: https://techoverflow.net/2024/07/23/how-to-fix-numpy-dtype-size-changed-may-indicate-binary-incompatibility-expected-96-from-c-header-got-88-from-pyobject/
|
|
12
|
+
## Changes in 0.2.7
|
|
48
13
|
|
|
49
|
-
|
|
50
|
-
|
|
14
|
+
- Sensitive information redaction now uses exact match positions on each line and merges overlaps, so it won't redact unrelated copies of a substring or leave part of an overlapping match exposed. Parenthesized phone area codes and signed coordinates are included in the matches.
|
|
15
|
+
- Profanity filtering supports the documented Finnish `FI` alias and punctuation in custom terms. Detections are in text order and the `Word` value preserves the original casing. Lines surrounded by `$` are no longer silently dropped.
|
|
16
|
+
- The bundled hate-speech models load on first use instead of at import time. This release requires Python 3.10 or newer.
|
|
17
|
+
- The release workflow builds and tests the wheel and source distribution before publishing both to PyPI with signed attestations.
|
|
51
18
|
|
|
52
19
|
## Changes in 0.2.6
|
|
53
20
|
|
|
@@ -117,16 +84,7 @@ pip install valx
|
|
|
117
84
|
|
|
118
85
|
## Supported Python Versions
|
|
119
86
|
|
|
120
|
-
ValX
|
|
121
|
-
|
|
122
|
-
- Python 3.6
|
|
123
|
-
- Python 3.7
|
|
124
|
-
- Python 3.8
|
|
125
|
-
- Python 3.9
|
|
126
|
-
- Python 3.10
|
|
127
|
-
- Python 3.11/Later (Preferred)
|
|
128
|
-
|
|
129
|
-
Please ensure that you have one of these Python versions installed before using ValX. ValX may not work as expected on lower versions of Python than the supported.
|
|
87
|
+
ValX requires Python 3.10 or newer. CI covers Python 3.10 through 3.14.
|
|
130
88
|
|
|
131
89
|
## Features
|
|
132
90
|
|
|
@@ -139,64 +97,38 @@ Please ensure that you have one of these Python versions installed before using
|
|
|
139
97
|
|
|
140
98
|
### List of supported languages for profanity detection and removal
|
|
141
99
|
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
- Kabyle
|
|
175
|
-
- KAB
|
|
176
|
-
- Korean
|
|
177
|
-
- KO
|
|
178
|
-
- Dutch
|
|
179
|
-
- NL
|
|
180
|
-
- Norwegian
|
|
181
|
-
- NO
|
|
182
|
-
- Polish
|
|
183
|
-
- PL
|
|
184
|
-
- Portuguese
|
|
185
|
-
- PT
|
|
186
|
-
- Russian
|
|
187
|
-
- RU
|
|
188
|
-
- Spanish
|
|
189
|
-
- ES
|
|
190
|
-
- Swedish
|
|
191
|
-
- SV
|
|
192
|
-
- Thai
|
|
193
|
-
- TH
|
|
194
|
-
- Klingon
|
|
195
|
-
- TLH
|
|
196
|
-
- Turkish
|
|
197
|
-
- TR
|
|
198
|
-
- Chinese
|
|
199
|
-
- ZH
|
|
100
|
+
Pass a language name or code from the table as `language`. Both are case-insensitive. Use `All` to combine all built-in lists.
|
|
101
|
+
|
|
102
|
+
| Language name | Code |
|
|
103
|
+
| --- | --- |
|
|
104
|
+
| Arabic | `AR` |
|
|
105
|
+
| Czech | `CS` |
|
|
106
|
+
| Danish | `DA` |
|
|
107
|
+
| German | `DE` |
|
|
108
|
+
| English | `EN` |
|
|
109
|
+
| Esperanto | `EO` |
|
|
110
|
+
| Persian | `FA` |
|
|
111
|
+
| Finnish | `FI` |
|
|
112
|
+
| Filipino | `FIL` |
|
|
113
|
+
| French | `FR` |
|
|
114
|
+
| French (CA) | `FR-CA-U-SD-CAQC` |
|
|
115
|
+
| Hindi | `HI` |
|
|
116
|
+
| Hungarian | `HU` |
|
|
117
|
+
| Italian | `IT` |
|
|
118
|
+
| Japanese | `JA` |
|
|
119
|
+
| Kabyle | `KAB` |
|
|
120
|
+
| Korean | `KO` |
|
|
121
|
+
| Dutch | `NL` |
|
|
122
|
+
| Norwegian | `NO` |
|
|
123
|
+
| Polish | `PL` |
|
|
124
|
+
| Portuguese | `PT` |
|
|
125
|
+
| Russian | `RU` |
|
|
126
|
+
| Spanish | `ES` |
|
|
127
|
+
| Swedish | `SV` |
|
|
128
|
+
| Thai | `TH` |
|
|
129
|
+
| Klingon | `TLH` |
|
|
130
|
+
| Turkish | `TR` |
|
|
131
|
+
| Chinese | `ZH` |
|
|
200
132
|
|
|
201
133
|
## Usage
|
|
202
134
|
|
|
@@ -242,7 +174,7 @@ results_custom_plus_english = detect_profanity(sample_text, language="English",
|
|
|
242
174
|
# Language will be "Custom + English"
|
|
243
175
|
# [
|
|
244
176
|
# {'Line': 1, 'Column': 15, 'Word': 'custombadword1', 'Language': 'Custom + English'},
|
|
245
|
-
# {'Line': 1, 'Column':
|
|
177
|
+
# {'Line': 1, 'Column': 39, 'Word': 'asshole', 'Language': 'Custom + English'}
|
|
246
178
|
# ]
|
|
247
179
|
print(results_custom_plus_english)
|
|
248
180
|
```
|
|
@@ -285,11 +217,11 @@ print(results_file_only)
|
|
|
285
217
|
|
|
286
218
|
**Output Format for `detect_profanity`**
|
|
287
219
|
|
|
288
|
-
The `detect_profanity` function returns
|
|
220
|
+
The `detect_profanity` function returns non-overlapping matches in text order. When two terms match at the same position, the longer one wins. Each dictionary includes:
|
|
289
221
|
|
|
290
222
|
- `"Line"`: The line number (1-indexed).
|
|
291
223
|
- `"Column"`: The column number (1-indexed) where the profanity starts.
|
|
292
|
-
- `"Word"`: The
|
|
224
|
+
- `"Word"`: The exact text that matched, with its original casing.
|
|
293
225
|
- `"Language"`: Indicates the source of the word list:
|
|
294
226
|
- `<LanguageName>` (e.g., "English"): If only a built-in language list was used.
|
|
295
227
|
- `"Custom"`: If `language=None` and only a `custom_words_list` was used.
|
|
@@ -323,26 +255,23 @@ print(cleaned_text)
|
|
|
323
255
|
|
|
324
256
|
The `load_profanity_words` function (used internally) also accepts `language` and `custom_words_list` if you need direct access to the word lists.
|
|
325
257
|
|
|
326
|
-
### Detect Sensitive Information
|
|
258
|
+
### Detect and Remove Sensitive Information
|
|
327
259
|
|
|
328
260
|
```python
|
|
329
|
-
from valx import detect_sensitive_information
|
|
330
|
-
|
|
331
|
-
# Detect sensitive information
|
|
332
|
-
detected_sensitive_info = detect_sensitive_information(sample_text)
|
|
333
|
-
```
|
|
261
|
+
from valx import detect_sensitive_information, remove_sensitive_information
|
|
334
262
|
|
|
335
|
-
|
|
336
|
-
|
|
263
|
+
lines = ["Contact help@example.com or call (555) 123-4567."]
|
|
264
|
+
print(detect_sensitive_information(lines, info_type="email"))
|
|
265
|
+
# [(1, 8, 'Email', 'help@example.com')]
|
|
337
266
|
|
|
338
|
-
|
|
267
|
+
print(remove_sensitive_information(lines, info_type=["email", "phone"]))
|
|
268
|
+
# ['Contact [SENSITIVE] or call [SENSITIVE].']
|
|
269
|
+
```
|
|
339
270
|
|
|
340
|
-
|
|
341
|
-
from valx import remove_sensitive_information
|
|
271
|
+
Pass one type, a list of types, or omit `info_type` to scan for all types. Detection reports a one-based line number and a **zero-based** column. Removal merges overlapping matches before inserting `[SENSITIVE]`.
|
|
342
272
|
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
```
|
|
273
|
+
> [!IMPORTANT]
|
|
274
|
+
> These are pattern-based filters, not a guarantee of anonymization. They can miss sensitive values or flag ordinary text. For example, `file_path` looks for filenames with an extension and can miss filenames containing spaces. Review redacted text before sharing it.
|
|
346
275
|
|
|
347
276
|
### Detect Hate Speech And Offensive Language
|
|
348
277
|
|
|
@@ -363,6 +292,16 @@ outcome_of_detection = detect_hate_speech("You are stupid.")
|
|
|
363
292
|
> [!NOTE]
|
|
364
293
|
> See our [official documentation](https://infinitode-docs.gitbook.io/documentation/package-documentation/valx-package-documentation) for more examples on how to use **ValX**.
|
|
365
294
|
|
|
295
|
+
### Model compatibility
|
|
296
|
+
|
|
297
|
+
Only the hate-speech functions load the bundled scikit-learn models. Scikit-learn can warn when loading a model saved with a different version, and compatibility across versions is not guaranteed. If you use scikit-learn 1.2.2, use NumPy 1.x with it. The rule-based profanity and sensitive-information filters do not load either model.
|
|
298
|
+
|
|
299
|
+
## PyPI provenance
|
|
300
|
+
|
|
301
|
+
Releases published by the Trusted Publishing workflow attach a [PEP 740 publish attestation](https://docs.pypi.org/attestations/publish/v1/) to each distribution file. The [PyPI Security tab](https://pypi.org/project/valx/#security) shows the status and lets you inspect the wheel and source archive separately. Older uploads are not retroactively attested.
|
|
302
|
+
|
|
303
|
+
For setup and verification steps, see the [maintainer release guide](https://github.com/Infinitode/ValX/blob/main/CONTRIBUTING.md#releasing-to-pypi).
|
|
304
|
+
|
|
366
305
|
## Contributing
|
|
367
306
|
|
|
368
307
|
Contributions are welcome! If you encounter any issues, have suggestions, or want to contribute to ValX, please open an issue or submit a pull request on [GitHub](https://github.com/infinitode/valx).
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "valx"
|
|
7
|
+
version = "0.2.7"
|
|
8
|
+
description = "Text filters for profanity, sensitive information, and offensive speech."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "LicenseRef-ValX-Modified-MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Infinitode Pty Ltd", email = "infinitode.ltd@gmail.com" }]
|
|
14
|
+
dependencies = ["packaging", "scikit-learn"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 5 - Production/Stable",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Programming Language :: Python :: 3.10",
|
|
20
|
+
"Programming Language :: Python :: 3.11",
|
|
21
|
+
"Programming Language :: Python :: 3.12",
|
|
22
|
+
"Programming Language :: Python :: 3.13",
|
|
23
|
+
"Programming Language :: Python :: 3.14",
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
[project.urls]
|
|
27
|
+
Repository = "https://github.com/Infinitode/ValX"
|
|
28
|
+
Issues = "https://github.com/Infinitode/ValX/issues"
|
|
29
|
+
Documentation = "https://infinitode-docs.gitbook.io/documentation/package-documentation/valx-package-documentation"
|
|
30
|
+
|
|
31
|
+
[tool.setuptools]
|
|
32
|
+
packages = ["valx"]
|
|
33
|
+
include-package-data = false
|
|
34
|
+
|
|
35
|
+
[tool.setuptools.package-data]
|
|
36
|
+
valx = ["models/*.pkl", "models/*.sav"]
|