valx 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- valx-0.1.0/LICENSE +7 -0
- valx-0.1.0/PKG-INFO +22 -0
- valx-0.1.0/README.md +77 -0
- valx-0.1.0/setup.cfg +4 -0
- valx-0.1.0/setup.py +26 -0
- valx-0.1.0/valx/__init__.py +123 -0
- valx-0.1.0/valx.egg-info/PKG-INFO +22 -0
- valx-0.1.0/valx.egg-info/SOURCES.txt +8 -0
- valx-0.1.0/valx.egg-info/dependency_links.txt +1 -0
- valx-0.1.0/valx.egg-info/top_level.txt +1 -0
valx-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
MIT License (Modified)
|
|
2
|
+
|
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to make derivative works based on the Software, provided that any substantial changes to the Software are clearly distinguished from the original work and are distributed under a different name.
|
|
4
|
+
|
|
5
|
+
The original copyright notice and disclaimer must be retained in all copies or substantial portions of the Software.
|
|
6
|
+
|
|
7
|
+
THE SOFTWARE IS PROVIDED "AS IS," WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT, OR OTHERWISE, ARISING FROM, OUT OF, OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
valx-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: valx
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: An open-source Python library for text cleaning tasks.
|
|
5
|
+
Home-page: https://github.com/infinitode/valx
|
|
6
|
+
Author: Infinitode Pty Ltd
|
|
7
|
+
Author-email: infinitode.ltd@gmail.com
|
|
8
|
+
Classifier: Development Status :: 3 - Alpha
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.6
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.7
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Requires-Python: >=3.6
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
|
|
22
|
+
An open-source Python library for data cleaning tasks. Includes profanity detection, and removal.
|
valx-0.1.0/README.md
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# ValX
|
|
2
|
+

|
|
3
|
+
[](https://github.com/infinitode/valx)
|
|
4
|
+

|
|
5
|
+

|
|
6
|
+

|
|
7
|
+
|
|
8
|
+
An open-source Python library for data cleaning tasks. Includes profanity detection, and removal.
|
|
9
|
+
|
|
10
|
+
## Installation
|
|
11
|
+
|
|
12
|
+
You can install ValX using pip:
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
pip install valx
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## Supported Python Versions
|
|
19
|
+
|
|
20
|
+
ValX supports the following Python versions:
|
|
21
|
+
|
|
22
|
+
- Python 3.6
|
|
23
|
+
- Python 3.7
|
|
24
|
+
- Python 3.8
|
|
25
|
+
- Python 3.9
|
|
26
|
+
- Python 3.10
|
|
27
|
+
- Python 3.11/Later (Preferred)
|
|
28
|
+
|
|
29
|
+
Please ensure that you have one of these Python versions installed before using ValX. ValX may not work as expected on lower versions of Python than the supported.
|
|
30
|
+
|
|
31
|
+
## Features
|
|
32
|
+
|
|
33
|
+
- **Profanity Detection**: Detect profane and NSFW words or terms.
|
|
34
|
+
- **Remove Profanity**: Remove profane and NSFW words or terms.
|
|
35
|
+
|
|
36
|
+
## Usage
|
|
37
|
+
|
|
38
|
+
### Detect Profanity
|
|
39
|
+
|
|
40
|
+
```python
|
|
41
|
+
from valx import detect_profanity
|
|
42
|
+
|
|
43
|
+
# Detect profanity
|
|
44
|
+
num_profanities = detect_profanity(sample_text, language='English')
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
### Remove Profanity
|
|
48
|
+
|
|
49
|
+
```python
|
|
50
|
+
from valx import remove_profanity
|
|
51
|
+
|
|
52
|
+
# Remove profanity
|
|
53
|
+
removed = remove_profanity(sample_text, "text_cleaned.txt", language="English")
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
## Contributing
|
|
57
|
+
|
|
58
|
+
Contributions are welcome! If you encounter any issues, have suggestions, or want to contribute to ValX, please open an issue or submit a pull request on [GitHub](https://github.com/infinitode/valx).
|
|
59
|
+
|
|
60
|
+
## License
|
|
61
|
+
|
|
62
|
+
PyWebScrapr is released under the terms of the **MIT License (Modified)**. Please see the [LICENSE](https://github.com/infinitode/pywebscrapr/blob/main/LICENSE) file for the full text.
|
|
63
|
+
|
|
64
|
+
### Derived licenses
|
|
65
|
+
---
|
|
66
|
+
ValX uses data from this GitHub repository:
|
|
67
|
+
https://github.com/LDNOOBW/List-of-Dirty-Naughty-Obscene-and-Otherwise-Bad-Words/
|
|
68
|
+
© 2012-2020 Shutterstock, Inc.
|
|
69
|
+
|
|
70
|
+
Creative Commons Attribution 4.0 International License:
|
|
71
|
+
https://github.com/LDNOOBW/List-of-Dirty-Naughty-Obscene-and-Otherwise-Bad-Words/blob/master/LICENSE
|
|
72
|
+
|
|
73
|
+
---
|
|
74
|
+
|
|
75
|
+
**Modified License Clause**
|
|
76
|
+
|
|
77
|
+
The modified license clause grants users the permission to make derivative works based on the ValX software. However, it requires any substantial changes to the software to be clearly distinguished from the original work and distributed under a different name.
|
valx-0.1.0/setup.cfg
ADDED
valx-0.1.0/setup.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
from setuptools import setup, find_packages
|
|
2
|
+
|
|
3
|
+
setup(
|
|
4
|
+
name='valx',
|
|
5
|
+
version='0.1.0',
|
|
6
|
+
author='Infinitode Pty Ltd',
|
|
7
|
+
author_email='infinitode.ltd@gmail.com',
|
|
8
|
+
description='An open-source Python library for text cleaning tasks.',
|
|
9
|
+
long_description='An open-source Python library for data cleaning tasks. Includes profanity detection, and removal.',
|
|
10
|
+
long_description_content_type='text/markdown',
|
|
11
|
+
url='https://github.com/infinitode/valx',
|
|
12
|
+
packages=find_packages(),
|
|
13
|
+
classifiers=[
|
|
14
|
+
'Development Status :: 3 - Alpha',
|
|
15
|
+
'Intended Audience :: Developers',
|
|
16
|
+
'License :: OSI Approved :: MIT License',
|
|
17
|
+
'Programming Language :: Python :: 3.6',
|
|
18
|
+
'Programming Language :: Python :: 3.7',
|
|
19
|
+
'Programming Language :: Python :: 3.8',
|
|
20
|
+
'Programming Language :: Python :: 3.9',
|
|
21
|
+
'Programming Language :: Python :: 3.10',
|
|
22
|
+
'Programming Language :: Python :: 3.11',
|
|
23
|
+
'Programming Language :: Python :: 3.12',
|
|
24
|
+
],
|
|
25
|
+
python_requires='>=3.6',
|
|
26
|
+
)
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import re
|
|
3
|
+
|
|
4
|
+
'''
|
|
5
|
+
ValX uses data from this GitHub repository:
|
|
6
|
+
https://github.com/LDNOOBW/List-of-Dirty-Naughty-Obscene-and-Otherwise-Bad-Words/
|
|
7
|
+
© 2012-2020 Shutterstock, Inc.
|
|
8
|
+
|
|
9
|
+
Creative Commons Attribution 4.0 International License:
|
|
10
|
+
https://github.com/LDNOOBW/List-of-Dirty-Naughty-Obscene-and-Otherwise-Bad-Words/blob/master/LICENSE
|
|
11
|
+
'''
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def load_profanity_words(language='English'):
|
|
15
|
+
"""
|
|
16
|
+
Load profanity words from the local text file.
|
|
17
|
+
|
|
18
|
+
Args:
|
|
19
|
+
language (str): The language for which to load profanity words. Defaults to 'English'.
|
|
20
|
+
|
|
21
|
+
Returns:
|
|
22
|
+
list: A list of profanity words for the specified language, or all languages if 'All' is specified.
|
|
23
|
+
"""
|
|
24
|
+
file_path = os.path.join(os.path.dirname(__file__), 'profanity_words.txt')
|
|
25
|
+
try:
|
|
26
|
+
with open(file_path, 'r', encoding='utf-8') as file:
|
|
27
|
+
profanity_lists = {}
|
|
28
|
+
current_language = None
|
|
29
|
+
current_words = []
|
|
30
|
+
for line in file:
|
|
31
|
+
line = line.strip()
|
|
32
|
+
if line.startswith("$") and line.endswith("$"):
|
|
33
|
+
if current_language:
|
|
34
|
+
profanity_lists[current_language] = current_words
|
|
35
|
+
current_words = []
|
|
36
|
+
current_language = line.strip("$")
|
|
37
|
+
else:
|
|
38
|
+
current_words.append(line)
|
|
39
|
+
if current_language:
|
|
40
|
+
profanity_lists[current_language] = current_words
|
|
41
|
+
|
|
42
|
+
if language == 'All':
|
|
43
|
+
all_profanity_words = []
|
|
44
|
+
for words in profanity_lists.values():
|
|
45
|
+
all_profanity_words.extend(words)
|
|
46
|
+
return all_profanity_words
|
|
47
|
+
else:
|
|
48
|
+
return profanity_lists.get(language, [])
|
|
49
|
+
except FileNotFoundError:
|
|
50
|
+
print("Failed to load profanity words file. Using default list.")
|
|
51
|
+
return [
|
|
52
|
+
'profane_word1',
|
|
53
|
+
'profane_word2',
|
|
54
|
+
'profane_word3'
|
|
55
|
+
]
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def detect_profanity(text_data, language='English', allowed_languages=None):
|
|
59
|
+
"""
|
|
60
|
+
Detect profanity in text data using regex.
|
|
61
|
+
|
|
62
|
+
Args:
|
|
63
|
+
text_data (list): A list of strings representing the text data to analyze.
|
|
64
|
+
language (str): The language for which to detect profanity. Defaults to 'English'.
|
|
65
|
+
allowed_languages (list): A list of languages to allow for detection. If provided, only profanity words for these languages will be considered.
|
|
66
|
+
|
|
67
|
+
Returns:
|
|
68
|
+
int: The total number of profanity detections.
|
|
69
|
+
"""
|
|
70
|
+
profanity_keywords = load_profanity_words(language)
|
|
71
|
+
detected = set() # Use a set to store unique occurrences
|
|
72
|
+
|
|
73
|
+
for i, line in enumerate(text_data):
|
|
74
|
+
# Skip lines with language markers
|
|
75
|
+
if line.startswith("$") and line.endswith("$"):
|
|
76
|
+
continue
|
|
77
|
+
|
|
78
|
+
# Detect profanity
|
|
79
|
+
for profanity in profanity_keywords:
|
|
80
|
+
matches = re.finditer(r'\b{}\b'.format(re.escape(profanity)), line, flags=re.IGNORECASE)
|
|
81
|
+
for match in matches:
|
|
82
|
+
detected.add((i + 1, match.start() + 1, language, profanity)) # Add to set
|
|
83
|
+
|
|
84
|
+
if detected:
|
|
85
|
+
print(f"Profanity detected:")
|
|
86
|
+
for line_num, col_num, lang, word in detected:
|
|
87
|
+
print(f"Language: {lang}, Line {line_num}, Column {col_num}: '{word}'")
|
|
88
|
+
|
|
89
|
+
return len(detected)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def remove_profanity(text_data, output_file=None, language='English'):
|
|
93
|
+
"""
|
|
94
|
+
Remove profanity from text data.
|
|
95
|
+
|
|
96
|
+
Args:
|
|
97
|
+
text_data (list): A list of strings representing the text data to clean.
|
|
98
|
+
output_file (str): The file path to write the cleaned data. If None, cleaned data is not written to a file.
|
|
99
|
+
language (str): The language for which to remove profanity. Defaults to 'English'.
|
|
100
|
+
|
|
101
|
+
Returns:
|
|
102
|
+
list: A list of strings representing the cleaned text data.
|
|
103
|
+
"""
|
|
104
|
+
profanity_keywords = load_profanity_words(language)
|
|
105
|
+
cleaned_data = []
|
|
106
|
+
|
|
107
|
+
for line in text_data:
|
|
108
|
+
# Skip lines with language markers
|
|
109
|
+
if line.startswith("$") and line.endswith("$"):
|
|
110
|
+
continue
|
|
111
|
+
|
|
112
|
+
# Remove profanity
|
|
113
|
+
cleaned_line = line
|
|
114
|
+
for profanity in profanity_keywords:
|
|
115
|
+
cleaned_line = re.sub(r'\b{}\b'.format(re.escape(profanity)), 'bad word', cleaned_line, flags=re.IGNORECASE)
|
|
116
|
+
cleaned_data.append(cleaned_line)
|
|
117
|
+
|
|
118
|
+
# Write cleaned data to output file if provided
|
|
119
|
+
if output_file:
|
|
120
|
+
with open(output_file, 'w', encoding='utf-8') as file:
|
|
121
|
+
file.write('\n'.join(cleaned_data))
|
|
122
|
+
|
|
123
|
+
return cleaned_data
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: valx
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: An open-source Python library for text cleaning tasks.
|
|
5
|
+
Home-page: https://github.com/infinitode/valx
|
|
6
|
+
Author: Infinitode Pty Ltd
|
|
7
|
+
Author-email: infinitode.ltd@gmail.com
|
|
8
|
+
Classifier: Development Status :: 3 - Alpha
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.6
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.7
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Requires-Python: >=3.6
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
|
|
22
|
+
An open-source Python library for data cleaning tasks. Includes profanity detection, and removal.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
valx
|