glin-profanity 2.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- glin_profanity-2.1.0/.gitignore +231 -0
- glin_profanity-2.1.0/PKG-INFO +193 -0
- glin_profanity-2.1.0/README.md +156 -0
- glin_profanity-2.1.0/glin_profanity/__init__.py +30 -0
- glin_profanity-2.1.0/glin_profanity/data/__init__.py +1 -0
- glin_profanity-2.1.0/glin_profanity/data/dictionary.py +89 -0
- glin_profanity-2.1.0/glin_profanity/filters/__init__.py +5 -0
- glin_profanity-2.1.0/glin_profanity/filters/filter.py +314 -0
- glin_profanity-2.1.0/glin_profanity/nlp/__init__.py +1 -0
- glin_profanity-2.1.0/glin_profanity/types/__init__.py +21 -0
- glin_profanity-2.1.0/glin_profanity/types/types.py +96 -0
- glin_profanity-2.1.0/pyproject.toml +157 -0
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
# Dependencies
|
|
2
|
+
node_modules/
|
|
3
|
+
__pycache__/
|
|
4
|
+
*.pyc
|
|
5
|
+
*.pyo
|
|
6
|
+
*.pyd
|
|
7
|
+
.Python
|
|
8
|
+
*.so
|
|
9
|
+
.venv/
|
|
10
|
+
venv/
|
|
11
|
+
env/
|
|
12
|
+
|
|
13
|
+
# Build outputs
|
|
14
|
+
lib/
|
|
15
|
+
dist/
|
|
16
|
+
build/
|
|
17
|
+
*.egg-info/
|
|
18
|
+
.eggs/
|
|
19
|
+
packages/*/lib/*
|
|
20
|
+
packages/*/dist/*
|
|
21
|
+
|
|
22
|
+
# TypeScript compiled output
|
|
23
|
+
*.tsbuildinfo
|
|
24
|
+
!src/index.d.ts
|
|
25
|
+
|
|
26
|
+
# Python specific
|
|
27
|
+
*.egg
|
|
28
|
+
.tox/
|
|
29
|
+
.nox/
|
|
30
|
+
.coverage.*
|
|
31
|
+
.cache
|
|
32
|
+
nosetests.xml
|
|
33
|
+
coverage.xml
|
|
34
|
+
*.cover
|
|
35
|
+
.hypothesis/
|
|
36
|
+
.pytest_cache/
|
|
37
|
+
.mypy_cache/
|
|
38
|
+
.dmypy.json
|
|
39
|
+
dmypy.json
|
|
40
|
+
.pyre/
|
|
41
|
+
# Logs
|
|
42
|
+
logs/
|
|
43
|
+
*.log
|
|
44
|
+
npm-debug.log*
|
|
45
|
+
yarn-debug.log*
|
|
46
|
+
yarn-error.log*
|
|
47
|
+
|
|
48
|
+
# Diagnostic reports
|
|
49
|
+
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
|
|
50
|
+
pids
|
|
51
|
+
*.pid
|
|
52
|
+
*.seed
|
|
53
|
+
*.pid.lock
|
|
54
|
+
|
|
55
|
+
# Directory for instrumented libs generated by jscoverage/JSCover
|
|
56
|
+
lib-cov/
|
|
57
|
+
|
|
58
|
+
# Coverage directory used by tools like istanbul
|
|
59
|
+
coverage/
|
|
60
|
+
|
|
61
|
+
# nyc test coverage
|
|
62
|
+
.nyc_output/
|
|
63
|
+
|
|
64
|
+
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
|
|
65
|
+
.grunt/
|
|
66
|
+
|
|
67
|
+
# Bower dependency directory (https://bower.io/)
|
|
68
|
+
bower_components/
|
|
69
|
+
|
|
70
|
+
# lock files
|
|
71
|
+
package-lock.json
|
|
72
|
+
yarn.lock
|
|
73
|
+
|
|
74
|
+
# dotenv environment variable files
|
|
75
|
+
.env
|
|
76
|
+
.env.local
|
|
77
|
+
.env.development.local
|
|
78
|
+
.env.test.local
|
|
79
|
+
.env.production.local
|
|
80
|
+
|
|
81
|
+
# parcel-bundler cache (https://parceljs.org/)
|
|
82
|
+
.cache
|
|
83
|
+
.parcel-cache
|
|
84
|
+
|
|
85
|
+
# next.js build output
|
|
86
|
+
.next
|
|
87
|
+
out
|
|
88
|
+
|
|
89
|
+
# nuxt.js build output
|
|
90
|
+
.nuxt
|
|
91
|
+
dist
|
|
92
|
+
|
|
93
|
+
# vuepress build output
|
|
94
|
+
.vuepress/dist
|
|
95
|
+
|
|
96
|
+
# Serverless directories
|
|
97
|
+
.serverless/
|
|
98
|
+
|
|
99
|
+
# FuseBox cache
|
|
100
|
+
.fusebox/
|
|
101
|
+
|
|
102
|
+
# DynamoDB Local files
|
|
103
|
+
.dynamodb/
|
|
104
|
+
|
|
105
|
+
# TernJS port file
|
|
106
|
+
.tern-port
|
|
107
|
+
|
|
108
|
+
# Stylint cache
|
|
109
|
+
.stylintcache
|
|
110
|
+
|
|
111
|
+
# Mac system files
|
|
112
|
+
.DS_Store
|
|
113
|
+
|
|
114
|
+
# Windows system files
|
|
115
|
+
Thumbs.db
|
|
116
|
+
|
|
117
|
+
# VSCode files
|
|
118
|
+
.vscode/
|
|
119
|
+
.history
|
|
120
|
+
|
|
121
|
+
# JetBrains IDE files
|
|
122
|
+
.idea/
|
|
123
|
+
*.iml
|
|
124
|
+
|
|
125
|
+
# Sentry CLI
|
|
126
|
+
.sentryclirc
|
|
127
|
+
|
|
128
|
+
# Firebase
|
|
129
|
+
.firebase/
|
|
130
|
+
|
|
131
|
+
# Generated by Visual Studio Code
|
|
132
|
+
.vscode/
|
|
133
|
+
|
|
134
|
+
# IntelliJ IDEA files
|
|
135
|
+
.idea/
|
|
136
|
+
*.iml
|
|
137
|
+
*.ipr
|
|
138
|
+
|
|
139
|
+
# IntelliJ IDEA log files
|
|
140
|
+
idea.log
|
|
141
|
+
|
|
142
|
+
# ESLint cache
|
|
143
|
+
.eslintcache
|
|
144
|
+
|
|
145
|
+
# Prettier cache
|
|
146
|
+
.prettiercache
|
|
147
|
+
|
|
148
|
+
# Storybook
|
|
149
|
+
.storybook/
|
|
150
|
+
node_modules/.cache/
|
|
151
|
+
|
|
152
|
+
# Jest
|
|
153
|
+
jest/
|
|
154
|
+
|
|
155
|
+
# Optional npm cache directory
|
|
156
|
+
.npm/
|
|
157
|
+
|
|
158
|
+
# Optional eslint cache
|
|
159
|
+
.eslintcache
|
|
160
|
+
|
|
161
|
+
# Optional REPL history
|
|
162
|
+
.node_repl_history
|
|
163
|
+
|
|
164
|
+
# dotenv environment variables file
|
|
165
|
+
.env
|
|
166
|
+
.env.test
|
|
167
|
+
.env.production
|
|
168
|
+
|
|
169
|
+
# node-waf configuration
|
|
170
|
+
.lock-wscript
|
|
171
|
+
|
|
172
|
+
# Compiled binary addons (https://nodejs.org/api/addons.html)
|
|
173
|
+
build/Release
|
|
174
|
+
|
|
175
|
+
# Coverage directory used by tools like istanbul
|
|
176
|
+
coverage/
|
|
177
|
+
|
|
178
|
+
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
|
|
179
|
+
.grunt/
|
|
180
|
+
|
|
181
|
+
# Bower dependency directory (https://bower.io/)
|
|
182
|
+
bower_components/
|
|
183
|
+
|
|
184
|
+
# lock files
|
|
185
|
+
package-lock.json
|
|
186
|
+
yarn.lock
|
|
187
|
+
|
|
188
|
+
# dotenv environment variable files
|
|
189
|
+
.env
|
|
190
|
+
.env.test
|
|
191
|
+
.env.production
|
|
192
|
+
|
|
193
|
+
# Optional npm cache directory
|
|
194
|
+
.npm/
|
|
195
|
+
|
|
196
|
+
# Optional eslint cache
|
|
197
|
+
.eslintcache
|
|
198
|
+
|
|
199
|
+
# Optional REPL history
|
|
200
|
+
.node_repl_history
|
|
201
|
+
|
|
202
|
+
# yarn v2
|
|
203
|
+
.yarn/cache
|
|
204
|
+
.yarn/unplugged
|
|
205
|
+
.yarn/build-state.yml
|
|
206
|
+
.yarn/install-state.gz
|
|
207
|
+
.pnp.*
|
|
208
|
+
|
|
209
|
+
# Optional stylelint cache
|
|
210
|
+
.stylelintcache
|
|
211
|
+
|
|
212
|
+
# Optional stylelint cache
|
|
213
|
+
.stylelintcache
|
|
214
|
+
|
|
215
|
+
# Optional comment: SvelteKit build output
|
|
216
|
+
.svelte-kit/
|
|
217
|
+
|
|
218
|
+
# TypeScript cache
|
|
219
|
+
*.tsbuildinfo
|
|
220
|
+
|
|
221
|
+
# coverage output for Jest
|
|
222
|
+
coverage/
|
|
223
|
+
|
|
224
|
+
# Cache and build artifacts
|
|
225
|
+
.cache/
|
|
226
|
+
build/
|
|
227
|
+
dist/
|
|
228
|
+
|
|
229
|
+
# IDE specific files
|
|
230
|
+
.idea/
|
|
231
|
+
.vscode/
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: glin-profanity
|
|
3
|
+
Version: 2.1.0
|
|
4
|
+
Summary: Glin-Profanity is a lightweight and efficient Python package designed to detect and filter profane language in text inputs across multiple languages.
|
|
5
|
+
Project-URL: Homepage, https://www.glincker.com/tools/glin-profanity
|
|
6
|
+
Project-URL: Documentation, https://github.com/GLINCKER/glin-profanity
|
|
7
|
+
Project-URL: Repository, https://github.com/GLINCKER/glin-profanity
|
|
8
|
+
Project-URL: Issues, https://github.com/GLINCKER/glin-profanity/issues
|
|
9
|
+
Author-email: glinr <contact@glincker.com>
|
|
10
|
+
Maintainer-email: glinr <contact@glincker.com>
|
|
11
|
+
License-Expression: MIT
|
|
12
|
+
Keywords: censorship,chat,comment,content,detection,filter,glin,glincker,language,moderation,nlp,profanity,social,text
|
|
13
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content
|
|
22
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
23
|
+
Classifier: Topic :: Text Processing :: Filters
|
|
24
|
+
Requires-Python: >=3.10
|
|
25
|
+
Requires-Dist: typing-extensions>=4.0.0
|
|
26
|
+
Provides-Extra: dev
|
|
27
|
+
Requires-Dist: black>=23.0.0; extra == 'dev'
|
|
28
|
+
Requires-Dist: isort>=5.0.0; extra == 'dev'
|
|
29
|
+
Requires-Dist: mypy>=1.0.0; extra == 'dev'
|
|
30
|
+
Requires-Dist: pytest-cov>=4.0.0; extra == 'dev'
|
|
31
|
+
Requires-Dist: pytest>=7.0.0; extra == 'dev'
|
|
32
|
+
Requires-Dist: ruff>=0.1.0; extra == 'dev'
|
|
33
|
+
Provides-Extra: test
|
|
34
|
+
Requires-Dist: pytest-cov>=4.0.0; extra == 'test'
|
|
35
|
+
Requires-Dist: pytest>=7.0.0; extra == 'test'
|
|
36
|
+
Description-Content-Type: text/markdown
|
|
37
|
+
|
|
38
|
+
# Glin-Profanity Python Package
|
|
39
|
+
|
|
40
|
+
A lightweight and efficient Python package designed to detect and filter profane language in text inputs across multiple languages.
|
|
41
|
+
|
|
42
|
+
## Features
|
|
43
|
+
|
|
44
|
+
- 🌍 **Multi-language Support**: Supports 25+ languages including English, Spanish, French, German, Arabic, Chinese, and many more
|
|
45
|
+
- 🎯 **Context-Aware Filtering**: Advanced context analysis to reduce false positives
|
|
46
|
+
- ⚙️ **Highly Configurable**: Customize word lists, severity levels, and filtering behavior
|
|
47
|
+
- 🚀 **High Performance**: Optimized for speed and efficiency
|
|
48
|
+
- 🔧 **Easy Integration**: Simple API that works with any Python application
|
|
49
|
+
- 📝 **TypeScript Compatible**: Mirrors the API of the TypeScript version
|
|
50
|
+
|
|
51
|
+
## Installation
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
pip install glin-profanity
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
## Quick Start
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
from glin_profanity import Filter
|
|
61
|
+
|
|
62
|
+
# Basic usage
|
|
63
|
+
filter_instance = Filter()
|
|
64
|
+
|
|
65
|
+
# Check if text contains profanity
|
|
66
|
+
if filter_instance.is_profane("This is a damn example"):
|
|
67
|
+
print("Profanity detected!")
|
|
68
|
+
|
|
69
|
+
# Get detailed results
|
|
70
|
+
result = filter_instance.check_profanity("This is a damn example")
|
|
71
|
+
print(result["profane_words"]) # ['damn']
|
|
72
|
+
print(result["contains_profanity"]) # True
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
## Configuration Options
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
from glin_profanity import Filter, SeverityLevel
|
|
79
|
+
|
|
80
|
+
# Advanced configuration
|
|
81
|
+
config = {
|
|
82
|
+
"languages": ["english", "spanish"], # Specific languages
|
|
83
|
+
"case_sensitive": False, # Case sensitivity
|
|
84
|
+
"word_boundaries": True, # Enforce word boundaries
|
|
85
|
+
"replace_with": "***", # Replacement text
|
|
86
|
+
"severity_levels": True, # Enable severity detection
|
|
87
|
+
"custom_words": ["badword"], # Add custom words
|
|
88
|
+
"ignore_words": ["exception"], # Ignore specific words
|
|
89
|
+
"allow_obfuscated_match": True, # Detect obfuscated text
|
|
90
|
+
"fuzzy_tolerance_level": 0.8, # Fuzzy matching threshold
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
filter_instance = Filter(config)
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
## API Reference
|
|
97
|
+
|
|
98
|
+
### Filter Class
|
|
99
|
+
|
|
100
|
+
#### `__init__(config: Optional[FilterConfig] = None)`
|
|
101
|
+
Initialize the filter with optional configuration.
|
|
102
|
+
|
|
103
|
+
#### `is_profane(text: str) -> bool`
|
|
104
|
+
Check if text contains profanity. Returns `True` if profanity is detected.
|
|
105
|
+
|
|
106
|
+
#### `check_profanity(text: str) -> CheckProfanityResult`
|
|
107
|
+
Perform comprehensive profanity analysis with detailed results.
|
|
108
|
+
|
|
109
|
+
#### `matches(word: str) -> bool`
|
|
110
|
+
Check if a single word matches profanity patterns. Alias for `is_profane()`.
|
|
111
|
+
|
|
112
|
+
#### `check_profanity_with_min_severity(text: str, min_severity: SeverityLevel) -> Dict`
|
|
113
|
+
Check profanity with minimum severity filtering.
|
|
114
|
+
|
|
115
|
+
### Types
|
|
116
|
+
|
|
117
|
+
#### `CheckProfanityResult`
|
|
118
|
+
```python
|
|
119
|
+
{
|
|
120
|
+
"contains_profanity": bool,
|
|
121
|
+
"profane_words": List[str],
|
|
122
|
+
"processed_text": Optional[str], # If replace_with is set
|
|
123
|
+
"severity_map": Optional[Dict], # If severity_levels is True
|
|
124
|
+
"matches": Optional[List[Match]], # Detailed match information
|
|
125
|
+
"context_score": Optional[float], # Context analysis score
|
|
126
|
+
"reason": Optional[str] # Analysis reason
|
|
127
|
+
}
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
#### `SeverityLevel`
|
|
131
|
+
- `SeverityLevel.EXACT`: Exact word match
|
|
132
|
+
- `SeverityLevel.FUZZY`: Fuzzy/approximate match
|
|
133
|
+
|
|
134
|
+
## Supported Languages
|
|
135
|
+
|
|
136
|
+
Arabic, Chinese, Czech, Danish, English, Esperanto, Finnish, French, German, Hindi, Hungarian, Italian, Japanese, Korean, Norwegian, Persian, Polish, Portuguese, Russian, Spanish, Swedish, Thai, Turkish
|
|
137
|
+
|
|
138
|
+
## Development
|
|
139
|
+
|
|
140
|
+
### Setup
|
|
141
|
+
|
|
142
|
+
```bash
|
|
143
|
+
# Clone the repository
|
|
144
|
+
git clone https://github.com/GLINCKER/glin-profanity
|
|
145
|
+
cd glin-profanity/packages/py
|
|
146
|
+
|
|
147
|
+
# Install with development dependencies
|
|
148
|
+
pip install -e ".[dev]"
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
### Testing
|
|
152
|
+
|
|
153
|
+
```bash
|
|
154
|
+
# Run tests
|
|
155
|
+
pytest
|
|
156
|
+
|
|
157
|
+
# Run tests with coverage
|
|
158
|
+
pytest --cov=glin_profanity
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
### Code Quality
|
|
162
|
+
|
|
163
|
+
```bash
|
|
164
|
+
# Format code
|
|
165
|
+
black glin_profanity tests
|
|
166
|
+
|
|
167
|
+
# Sort imports
|
|
168
|
+
isort glin_profanity tests
|
|
169
|
+
|
|
170
|
+
# Type checking
|
|
171
|
+
mypy glin_profanity
|
|
172
|
+
|
|
173
|
+
# Linting
|
|
174
|
+
ruff check glin_profanity tests
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
## License
|
|
178
|
+
|
|
179
|
+
This project is licensed under the MIT License - see the [LICENSE](../../LICENSE) file for details.
|
|
180
|
+
|
|
181
|
+
## Contributing
|
|
182
|
+
|
|
183
|
+
Contributions are welcome! Please read our [Contributing Guide](../../CONTRIBUTING.md) for details on our code of conduct and the process for submitting pull requests.
|
|
184
|
+
|
|
185
|
+
## Support
|
|
186
|
+
|
|
187
|
+
- 📖 [Documentation](https://github.com/GLINCKER/glin-profanity)
|
|
188
|
+
- 🐛 [Issue Tracker](https://github.com/GLINCKER/glin-profanity/issues)
|
|
189
|
+
- 💬 [Discussions](https://github.com/GLINCKER/glin-profanity/discussions)
|
|
190
|
+
|
|
191
|
+
## Changelog
|
|
192
|
+
|
|
193
|
+
See [CHANGELOG.md](../../CHANGELOG.md) for a list of changes and updates.
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
# Glin-Profanity Python Package
|
|
2
|
+
|
|
3
|
+
A lightweight and efficient Python package designed to detect and filter profane language in text inputs across multiple languages.
|
|
4
|
+
|
|
5
|
+
## Features
|
|
6
|
+
|
|
7
|
+
- 🌍 **Multi-language Support**: Supports 25+ languages including English, Spanish, French, German, Arabic, Chinese, and many more
|
|
8
|
+
- 🎯 **Context-Aware Filtering**: Advanced context analysis to reduce false positives
|
|
9
|
+
- ⚙️ **Highly Configurable**: Customize word lists, severity levels, and filtering behavior
|
|
10
|
+
- 🚀 **High Performance**: Optimized for speed and efficiency
|
|
11
|
+
- 🔧 **Easy Integration**: Simple API that works with any Python application
|
|
12
|
+
- 📝 **TypeScript Compatible**: Mirrors the API of the TypeScript version
|
|
13
|
+
|
|
14
|
+
## Installation
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
pip install glin-profanity
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
## Quick Start
|
|
21
|
+
|
|
22
|
+
```python
|
|
23
|
+
from glin_profanity import Filter
|
|
24
|
+
|
|
25
|
+
# Basic usage
|
|
26
|
+
filter_instance = Filter()
|
|
27
|
+
|
|
28
|
+
# Check if text contains profanity
|
|
29
|
+
if filter_instance.is_profane("This is a damn example"):
|
|
30
|
+
print("Profanity detected!")
|
|
31
|
+
|
|
32
|
+
# Get detailed results
|
|
33
|
+
result = filter_instance.check_profanity("This is a damn example")
|
|
34
|
+
print(result["profane_words"]) # ['damn']
|
|
35
|
+
print(result["contains_profanity"]) # True
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
## Configuration Options
|
|
39
|
+
|
|
40
|
+
```python
|
|
41
|
+
from glin_profanity import Filter, SeverityLevel
|
|
42
|
+
|
|
43
|
+
# Advanced configuration
|
|
44
|
+
config = {
|
|
45
|
+
"languages": ["english", "spanish"], # Specific languages
|
|
46
|
+
"case_sensitive": False, # Case sensitivity
|
|
47
|
+
"word_boundaries": True, # Enforce word boundaries
|
|
48
|
+
"replace_with": "***", # Replacement text
|
|
49
|
+
"severity_levels": True, # Enable severity detection
|
|
50
|
+
"custom_words": ["badword"], # Add custom words
|
|
51
|
+
"ignore_words": ["exception"], # Ignore specific words
|
|
52
|
+
"allow_obfuscated_match": True, # Detect obfuscated text
|
|
53
|
+
"fuzzy_tolerance_level": 0.8, # Fuzzy matching threshold
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
filter_instance = Filter(config)
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## API Reference
|
|
60
|
+
|
|
61
|
+
### Filter Class
|
|
62
|
+
|
|
63
|
+
#### `__init__(config: Optional[FilterConfig] = None)`
|
|
64
|
+
Initialize the filter with optional configuration.
|
|
65
|
+
|
|
66
|
+
#### `is_profane(text: str) -> bool`
|
|
67
|
+
Check if text contains profanity. Returns `True` if profanity is detected.
|
|
68
|
+
|
|
69
|
+
#### `check_profanity(text: str) -> CheckProfanityResult`
|
|
70
|
+
Perform comprehensive profanity analysis with detailed results.
|
|
71
|
+
|
|
72
|
+
#### `matches(word: str) -> bool`
|
|
73
|
+
Check if a single word matches profanity patterns. Alias for `is_profane()`.
|
|
74
|
+
|
|
75
|
+
#### `check_profanity_with_min_severity(text: str, min_severity: SeverityLevel) -> Dict`
|
|
76
|
+
Check profanity with minimum severity filtering.
|
|
77
|
+
|
|
78
|
+
### Types
|
|
79
|
+
|
|
80
|
+
#### `CheckProfanityResult`
|
|
81
|
+
```python
|
|
82
|
+
{
|
|
83
|
+
"contains_profanity": bool,
|
|
84
|
+
"profane_words": List[str],
|
|
85
|
+
"processed_text": Optional[str], # If replace_with is set
|
|
86
|
+
"severity_map": Optional[Dict], # If severity_levels is True
|
|
87
|
+
"matches": Optional[List[Match]], # Detailed match information
|
|
88
|
+
"context_score": Optional[float], # Context analysis score
|
|
89
|
+
"reason": Optional[str] # Analysis reason
|
|
90
|
+
}
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
#### `SeverityLevel`
|
|
94
|
+
- `SeverityLevel.EXACT`: Exact word match
|
|
95
|
+
- `SeverityLevel.FUZZY`: Fuzzy/approximate match
|
|
96
|
+
|
|
97
|
+
## Supported Languages
|
|
98
|
+
|
|
99
|
+
Arabic, Chinese, Czech, Danish, English, Esperanto, Finnish, French, German, Hindi, Hungarian, Italian, Japanese, Korean, Norwegian, Persian, Polish, Portuguese, Russian, Spanish, Swedish, Thai, Turkish
|
|
100
|
+
|
|
101
|
+
## Development
|
|
102
|
+
|
|
103
|
+
### Setup
|
|
104
|
+
|
|
105
|
+
```bash
|
|
106
|
+
# Clone the repository
|
|
107
|
+
git clone https://github.com/GLINCKER/glin-profanity
|
|
108
|
+
cd glin-profanity/packages/py
|
|
109
|
+
|
|
110
|
+
# Install with development dependencies
|
|
111
|
+
pip install -e ".[dev]"
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
### Testing
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
# Run tests
|
|
118
|
+
pytest
|
|
119
|
+
|
|
120
|
+
# Run tests with coverage
|
|
121
|
+
pytest --cov=glin_profanity
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
### Code Quality
|
|
125
|
+
|
|
126
|
+
```bash
|
|
127
|
+
# Format code
|
|
128
|
+
black glin_profanity tests
|
|
129
|
+
|
|
130
|
+
# Sort imports
|
|
131
|
+
isort glin_profanity tests
|
|
132
|
+
|
|
133
|
+
# Type checking
|
|
134
|
+
mypy glin_profanity
|
|
135
|
+
|
|
136
|
+
# Linting
|
|
137
|
+
ruff check glin_profanity tests
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
## License
|
|
141
|
+
|
|
142
|
+
This project is licensed under the MIT License - see the [LICENSE](../../LICENSE) file for details.
|
|
143
|
+
|
|
144
|
+
## Contributing
|
|
145
|
+
|
|
146
|
+
Contributions are welcome! Please read our [Contributing Guide](../../CONTRIBUTING.md) for details on our code of conduct and the process for submitting pull requests.
|
|
147
|
+
|
|
148
|
+
## Support
|
|
149
|
+
|
|
150
|
+
- 📖 [Documentation](https://github.com/GLINCKER/glin-profanity)
|
|
151
|
+
- 🐛 [Issue Tracker](https://github.com/GLINCKER/glin-profanity/issues)
|
|
152
|
+
- 💬 [Discussions](https://github.com/GLINCKER/glin-profanity/discussions)
|
|
153
|
+
|
|
154
|
+
## Changelog
|
|
155
|
+
|
|
156
|
+
See [CHANGELOG.md](../../CHANGELOG.md) for a list of changes and updates.
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Glin-Profanity: A lightweight and efficient Python package for profanity detection.
|
|
3
|
+
|
|
4
|
+
Provides multi-language support, context-aware filtering, and customizable
|
|
5
|
+
configurations for detecting and filtering profane language in text inputs.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
__version__ = "2.1.0"
|
|
9
|
+
__author__ = "glinr"
|
|
10
|
+
__email__ = "contact@glincker.com"
|
|
11
|
+
|
|
12
|
+
from .filters.filter import Filter
|
|
13
|
+
from .types.types import (
|
|
14
|
+
CheckProfanityResult,
|
|
15
|
+
FilterConfig,
|
|
16
|
+
FilteredProfanityResult,
|
|
17
|
+
Language,
|
|
18
|
+
Match,
|
|
19
|
+
SeverityLevel,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
__all__ = [
|
|
23
|
+
"CheckProfanityResult",
|
|
24
|
+
"Filter",
|
|
25
|
+
"FilterConfig",
|
|
26
|
+
"FilteredProfanityResult",
|
|
27
|
+
"Language",
|
|
28
|
+
"Match",
|
|
29
|
+
"SeverityLevel",
|
|
30
|
+
]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Dictionary data loader for profanity detection."""
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"""Dictionary loader for profanity word lists."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from glin_profanity.types.types import Language
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class DictionaryLoader:
|
|
10
|
+
"""Loads profanity dictionaries from shared JSON files."""
|
|
11
|
+
|
|
12
|
+
def __init__(self) -> None:
|
|
13
|
+
"""Initialize dictionary loader."""
|
|
14
|
+
# Get path to shared dictionaries (relative to this file)
|
|
15
|
+
current_dir = Path(__file__).parent
|
|
16
|
+
self._dict_path = (
|
|
17
|
+
current_dir.parent.parent.parent.parent / "shared" / "dictionaries"
|
|
18
|
+
)
|
|
19
|
+
self._dictionaries: dict[str, list[str]] = {}
|
|
20
|
+
self._load_dictionaries()
|
|
21
|
+
|
|
22
|
+
def _raise_format_error(self, filename: str) -> None:
|
|
23
|
+
"""Raise a ValueError for unexpected file format."""
|
|
24
|
+
msg = f"Unexpected format in {filename}"
|
|
25
|
+
raise ValueError(msg)
|
|
26
|
+
|
|
27
|
+
def _load_dictionaries(self) -> None:
|
|
28
|
+
"""Load all dictionary files from shared directory."""
|
|
29
|
+
language_files = {
|
|
30
|
+
"arabic": "arabic.json",
|
|
31
|
+
"chinese": "chinese.json",
|
|
32
|
+
"czech": "czech.json",
|
|
33
|
+
"danish": "danish.json",
|
|
34
|
+
"english": "english.json",
|
|
35
|
+
"esperanto": "esperanto.json",
|
|
36
|
+
"finnish": "finnish.json",
|
|
37
|
+
"french": "french.json",
|
|
38
|
+
"german": "german.json",
|
|
39
|
+
"hindi": "hindi.json",
|
|
40
|
+
"hungarian": "hungarian.json",
|
|
41
|
+
"italian": "italian.json",
|
|
42
|
+
"japanese": "japanese.json",
|
|
43
|
+
"korean": "korean.json",
|
|
44
|
+
"norwegian": "Norwegian.json",
|
|
45
|
+
"persian": "persian.json",
|
|
46
|
+
"polish": "polish.json",
|
|
47
|
+
"portuguese": "portuguese.json",
|
|
48
|
+
"russian": "russian.json",
|
|
49
|
+
"spanish": "spanish.json",
|
|
50
|
+
"swedish": "swedish.json",
|
|
51
|
+
"thai": "thai.json",
|
|
52
|
+
"turkish": "turkish.json",
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
for language, filename in language_files.items():
|
|
56
|
+
file_path = self._dict_path / filename
|
|
57
|
+
try:
|
|
58
|
+
with file_path.open(encoding="utf-8") as f:
|
|
59
|
+
data = json.load(f)
|
|
60
|
+
# Handle both {"words": [...]} and [...] formats
|
|
61
|
+
if isinstance(data, dict) and "words" in data:
|
|
62
|
+
self._dictionaries[language] = data["words"]
|
|
63
|
+
elif isinstance(data, list):
|
|
64
|
+
self._dictionaries[language] = data
|
|
65
|
+
else:
|
|
66
|
+
self._raise_format_error(filename)
|
|
67
|
+
except (FileNotFoundError, json.JSONDecodeError, ValueError) as e:
|
|
68
|
+
print(f"Warning: Could not load {filename}: {e}") # noqa: T201
|
|
69
|
+
self._dictionaries[language] = []
|
|
70
|
+
|
|
71
|
+
def get_words(self, language: Language) -> list[str]:
|
|
72
|
+
"""Get words for a specific language."""
|
|
73
|
+
return self._dictionaries.get(language, [])
|
|
74
|
+
|
|
75
|
+
def get_all_words(self) -> list[str]:
|
|
76
|
+
"""Get all words from all languages."""
|
|
77
|
+
all_words = []
|
|
78
|
+
for words in self._dictionaries.values():
|
|
79
|
+
all_words.extend(words)
|
|
80
|
+
return list(set(all_words)) # Remove duplicates
|
|
81
|
+
|
|
82
|
+
@property
|
|
83
|
+
def available_languages(self) -> list[str]:
|
|
84
|
+
"""Get list of available languages."""
|
|
85
|
+
return list(self._dictionaries.keys())
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
# Global instance
|
|
89
|
+
dictionary = DictionaryLoader()
|
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
"""Main Filter class for profanity detection and filtering."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
|
|
5
|
+
from glin_profanity.data.dictionary import dictionary
|
|
6
|
+
from glin_profanity.types.types import (
|
|
7
|
+
CheckProfanityResult,
|
|
8
|
+
FilterConfig,
|
|
9
|
+
Match,
|
|
10
|
+
SeverityLevel,
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Filter:
|
|
15
|
+
"""
|
|
16
|
+
Main profanity filter class.
|
|
17
|
+
|
|
18
|
+
Provides functionality to detect and filter profane language in text
|
|
19
|
+
with support for multiple languages, custom configurations, and
|
|
20
|
+
context-aware filtering.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
def __init__(self, config: FilterConfig | None = None) -> None:
|
|
24
|
+
"""
|
|
25
|
+
Initialize the profanity filter.
|
|
26
|
+
|
|
27
|
+
Args:
|
|
28
|
+
config: Configuration options for the filter
|
|
29
|
+
"""
|
|
30
|
+
config = config or {}
|
|
31
|
+
|
|
32
|
+
# Basic configuration
|
|
33
|
+
self.case_sensitive = config.get("case_sensitive", False)
|
|
34
|
+
self.allow_obfuscated_match = config.get("allow_obfuscated_match", False)
|
|
35
|
+
self.word_boundaries = config.get(
|
|
36
|
+
"word_boundaries", not self.allow_obfuscated_match
|
|
37
|
+
)
|
|
38
|
+
self.replace_with = config.get("replace_with")
|
|
39
|
+
self.severity_levels = config.get("severity_levels", False)
|
|
40
|
+
self.log_profanity = config.get("log_profanity", False)
|
|
41
|
+
self.fuzzy_tolerance_level = config.get("fuzzy_tolerance_level", 0.8)
|
|
42
|
+
|
|
43
|
+
# Context-aware configuration
|
|
44
|
+
self.enable_context_aware = config.get("enable_context_aware", False)
|
|
45
|
+
self.context_window = config.get("context_window", 3)
|
|
46
|
+
self.confidence_threshold = config.get("confidence_threshold", 0.7)
|
|
47
|
+
|
|
48
|
+
# Initialize word sets
|
|
49
|
+
ignore_words_list = config.get("ignore_words", [])
|
|
50
|
+
self.ignore_words: set[str] = {word.lower() for word in ignore_words_list}
|
|
51
|
+
|
|
52
|
+
# Load dictionary words
|
|
53
|
+
self._load_words(config)
|
|
54
|
+
|
|
55
|
+
def _load_words(self, config: FilterConfig) -> None:
|
|
56
|
+
"""Load profanity words based on configuration."""
|
|
57
|
+
words: list[str] = []
|
|
58
|
+
|
|
59
|
+
if config.get("all_languages", False):
|
|
60
|
+
words = dictionary.get_all_words()
|
|
61
|
+
else:
|
|
62
|
+
languages = config.get("languages", ["english"])
|
|
63
|
+
if languages:
|
|
64
|
+
for lang in languages:
|
|
65
|
+
words.extend(dictionary.get_words(lang))
|
|
66
|
+
|
|
67
|
+
# Add custom words if provided
|
|
68
|
+
custom_words = config.get("custom_words")
|
|
69
|
+
if custom_words:
|
|
70
|
+
words.extend(custom_words)
|
|
71
|
+
|
|
72
|
+
# Store as set for faster lookup
|
|
73
|
+
self.words: set[str] = {word.lower() for word in words}
|
|
74
|
+
|
|
75
|
+
def _debug_log(self, *args: object) -> None:
|
|
76
|
+
"""Log debug information if logging is enabled."""
|
|
77
|
+
if self.log_profanity:
|
|
78
|
+
print("[glin-profanity]", *args) # noqa: T201
|
|
79
|
+
|
|
80
|
+
def _normalize_obfuscated(self, text: str) -> str:
|
|
81
|
+
"""Normalize obfuscated text by replacing common character substitutions."""
|
|
82
|
+
# Remove repeated characters (e.g., "hiiiii" -> "hii")
|
|
83
|
+
normalized = re.sub(r"([a-zA-Z])\1{1,}", r"\1\1", text)
|
|
84
|
+
|
|
85
|
+
# Character substitution map
|
|
86
|
+
char_map = {
|
|
87
|
+
"@": "a",
|
|
88
|
+
"$": "s",
|
|
89
|
+
"!": "i",
|
|
90
|
+
"1": "i",
|
|
91
|
+
"*": "",
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
for char, replacement in char_map.items():
|
|
95
|
+
normalized = normalized.replace(char, replacement)
|
|
96
|
+
|
|
97
|
+
return normalized
|
|
98
|
+
|
|
99
|
+
def _get_regex(self, word: str) -> re.Pattern[str]:
|
|
100
|
+
"""Create regex pattern for word matching."""
|
|
101
|
+
flags = 0 if self.case_sensitive else re.IGNORECASE
|
|
102
|
+
escaped_word = re.escape(word)
|
|
103
|
+
|
|
104
|
+
pattern = rf"\b{escaped_word}\b" if self.word_boundaries else escaped_word
|
|
105
|
+
|
|
106
|
+
return re.compile(pattern, flags)
|
|
107
|
+
|
|
108
|
+
def _is_fuzzy_tolerance_match(self, word: str, text: str) -> bool:
|
|
109
|
+
"""Check if word matches text within fuzzy tolerance."""
|
|
110
|
+
simplified_text = re.sub(r"[^a-z]", "", text.lower())
|
|
111
|
+
simplified_word = word.lower()
|
|
112
|
+
|
|
113
|
+
# If word boundaries are enabled, don't do fuzzy matching
|
|
114
|
+
# that could match across word boundaries
|
|
115
|
+
if self.word_boundaries:
|
|
116
|
+
# Only do fuzzy matching if the word appears as a separate token
|
|
117
|
+
words_in_text = re.findall(r"\b\w+\b", text.lower())
|
|
118
|
+
for text_word in words_in_text:
|
|
119
|
+
if self._fuzzy_match_single_word(simplified_word, text_word):
|
|
120
|
+
return True
|
|
121
|
+
return False
|
|
122
|
+
|
|
123
|
+
# Original fuzzy matching for non-word-boundary mode
|
|
124
|
+
match_count = 0
|
|
125
|
+
index = 0
|
|
126
|
+
|
|
127
|
+
for char in simplified_text:
|
|
128
|
+
if index < len(simplified_word) and char == simplified_word[index]:
|
|
129
|
+
match_count += 1
|
|
130
|
+
index += 1
|
|
131
|
+
|
|
132
|
+
score = match_count / len(simplified_word) if simplified_word else 0
|
|
133
|
+
return score >= self.fuzzy_tolerance_level
|
|
134
|
+
|
|
135
|
+
def _fuzzy_match_single_word(self, pattern_word: str, text_word: str) -> bool:
|
|
136
|
+
"""Check if a single word matches the pattern with fuzzy tolerance."""
|
|
137
|
+
# For word boundary mode, require a more exact match
|
|
138
|
+
# The pattern word should be roughly the same length as the text word
|
|
139
|
+
if abs(len(pattern_word) - len(text_word)) > max(1, len(pattern_word) // 2):
|
|
140
|
+
return False
|
|
141
|
+
|
|
142
|
+
match_count = 0
|
|
143
|
+
index = 0
|
|
144
|
+
|
|
145
|
+
for char in text_word:
|
|
146
|
+
if index < len(pattern_word) and char == pattern_word[index]:
|
|
147
|
+
match_count += 1
|
|
148
|
+
index += 1
|
|
149
|
+
|
|
150
|
+
score = match_count / len(pattern_word) if pattern_word else 0
|
|
151
|
+
return score >= self.fuzzy_tolerance_level
|
|
152
|
+
|
|
153
|
+
def _evaluate_severity(self, word: str, text: str) -> SeverityLevel | None:
|
|
154
|
+
"""Evaluate the severity level of a match."""
|
|
155
|
+
regex = self._get_regex(word)
|
|
156
|
+
|
|
157
|
+
if regex.search(text):
|
|
158
|
+
return SeverityLevel.EXACT
|
|
159
|
+
if self._is_fuzzy_tolerance_match(word, text):
|
|
160
|
+
return SeverityLevel.FUZZY
|
|
161
|
+
|
|
162
|
+
return None
|
|
163
|
+
|
|
164
|
+
def is_profane(self, value: str) -> bool:
|
|
165
|
+
"""
|
|
166
|
+
Check if text contains profanity.
|
|
167
|
+
|
|
168
|
+
Args:
|
|
169
|
+
value: Text to check
|
|
170
|
+
|
|
171
|
+
Returns:
|
|
172
|
+
True if profanity is detected, False otherwise
|
|
173
|
+
"""
|
|
174
|
+
input_text = (
|
|
175
|
+
self._normalize_obfuscated(value) if self.allow_obfuscated_match else value
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
for word in self.words:
|
|
179
|
+
if (
|
|
180
|
+
word.lower() not in self.ignore_words
|
|
181
|
+
and self._evaluate_severity(word, input_text) is not None
|
|
182
|
+
):
|
|
183
|
+
return True
|
|
184
|
+
|
|
185
|
+
return False
|
|
186
|
+
|
|
187
|
+
def matches(self, word: str) -> bool:
|
|
188
|
+
"""
|
|
189
|
+
Check if a single word matches profanity patterns.
|
|
190
|
+
|
|
191
|
+
Args:
|
|
192
|
+
word: Word to check
|
|
193
|
+
|
|
194
|
+
Returns:
|
|
195
|
+
True if word matches profanity, False otherwise
|
|
196
|
+
"""
|
|
197
|
+
return self.is_profane(word)
|
|
198
|
+
|
|
199
|
+
def check_profanity(self, text: str) -> CheckProfanityResult:
|
|
200
|
+
"""
|
|
201
|
+
Comprehensive profanity check with detailed results.
|
|
202
|
+
|
|
203
|
+
Args:
|
|
204
|
+
text: Text to analyze
|
|
205
|
+
|
|
206
|
+
Returns:
|
|
207
|
+
Detailed results of profanity analysis
|
|
208
|
+
"""
|
|
209
|
+
input_text = (
|
|
210
|
+
self._normalize_obfuscated(text) if self.allow_obfuscated_match else text
|
|
211
|
+
)
|
|
212
|
+
input_lower = input_text.lower()
|
|
213
|
+
|
|
214
|
+
profane_words: list[str] = []
|
|
215
|
+
severity_map: dict[str, SeverityLevel] = {}
|
|
216
|
+
matches: list[Match] = []
|
|
217
|
+
|
|
218
|
+
# Check each word in dictionary
|
|
219
|
+
for dict_word in self.words:
|
|
220
|
+
if dict_word.lower() in self.ignore_words:
|
|
221
|
+
continue
|
|
222
|
+
|
|
223
|
+
severity = self._evaluate_severity(dict_word, input_text)
|
|
224
|
+
if severity is not None:
|
|
225
|
+
regex = self._get_regex(dict_word)
|
|
226
|
+
|
|
227
|
+
# Find all matches
|
|
228
|
+
for match in regex.finditer(input_text):
|
|
229
|
+
matched_word = match.group(0)
|
|
230
|
+
match_index = match.start()
|
|
231
|
+
|
|
232
|
+
profane_words.append(matched_word)
|
|
233
|
+
severity_map[matched_word] = severity
|
|
234
|
+
|
|
235
|
+
# Create match object
|
|
236
|
+
match_obj: Match = {
|
|
237
|
+
"word": matched_word,
|
|
238
|
+
"index": match_index,
|
|
239
|
+
"severity": severity,
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
# TODO: Add context analysis when implemented
|
|
243
|
+
matches.append(match_obj)
|
|
244
|
+
|
|
245
|
+
# Log detected profanity
|
|
246
|
+
if profane_words:
|
|
247
|
+
self._debug_log("Detected:", profane_words)
|
|
248
|
+
|
|
249
|
+
# Process text replacement if configured
|
|
250
|
+
processed_text = text
|
|
251
|
+
if self.replace_with and profane_words:
|
|
252
|
+
unique_words = list(set(profane_words))
|
|
253
|
+
for word in unique_words:
|
|
254
|
+
escaped = re.escape(word)
|
|
255
|
+
if self.word_boundaries:
|
|
256
|
+
replacement_regex = re.compile(rf"\b{escaped}\b", re.IGNORECASE)
|
|
257
|
+
else:
|
|
258
|
+
replacement_regex = re.compile(escaped, re.IGNORECASE)
|
|
259
|
+
processed_text = replacement_regex.sub(
|
|
260
|
+
self.replace_with, processed_text
|
|
261
|
+
)
|
|
262
|
+
|
|
263
|
+
# Build result
|
|
264
|
+
result: CheckProfanityResult = {
|
|
265
|
+
"contains_profanity": len(profane_words) > 0,
|
|
266
|
+
"profane_words": list(set(profane_words)),
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
if self.replace_with:
|
|
270
|
+
result["processed_text"] = processed_text
|
|
271
|
+
|
|
272
|
+
if self.severity_levels and severity_map:
|
|
273
|
+
result["severity_map"] = severity_map
|
|
274
|
+
|
|
275
|
+
if matches:
|
|
276
|
+
result["matches"] = matches
|
|
277
|
+
|
|
278
|
+
result["reason"] = (
|
|
279
|
+
f"Found {len(matches)} potential profanity matches"
|
|
280
|
+
if matches
|
|
281
|
+
else "No profanity detected"
|
|
282
|
+
)
|
|
283
|
+
|
|
284
|
+
return result
|
|
285
|
+
|
|
286
|
+
def check_profanity_with_min_severity(
|
|
287
|
+
self, text: str, min_severity: SeverityLevel = SeverityLevel.EXACT
|
|
288
|
+
) -> dict[str, object]:
|
|
289
|
+
"""
|
|
290
|
+
Check profanity with minimum severity filtering.
|
|
291
|
+
|
|
292
|
+
Args:
|
|
293
|
+
text: Text to analyze
|
|
294
|
+
min_severity: Minimum severity level to include
|
|
295
|
+
|
|
296
|
+
Returns:
|
|
297
|
+
Dictionary with filtered words and full result
|
|
298
|
+
"""
|
|
299
|
+
result = self.check_profanity(text)
|
|
300
|
+
|
|
301
|
+
filtered_words = []
|
|
302
|
+
severity_map = result.get("severity_map")
|
|
303
|
+
profane_words = result.get("profane_words")
|
|
304
|
+
if severity_map and profane_words:
|
|
305
|
+
filtered_words = [
|
|
306
|
+
word
|
|
307
|
+
for word in profane_words
|
|
308
|
+
if severity_map.get(word, 0) >= min_severity
|
|
309
|
+
]
|
|
310
|
+
|
|
311
|
+
return {
|
|
312
|
+
"filtered_words": filtered_words,
|
|
313
|
+
"result": result,
|
|
314
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""NLP components for glin-profanity."""
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""Type definitions for glin-profanity."""
|
|
2
|
+
|
|
3
|
+
from .types import (
|
|
4
|
+
CheckProfanityResult,
|
|
5
|
+
ContextAwareConfig,
|
|
6
|
+
FilterConfig,
|
|
7
|
+
FilteredProfanityResult,
|
|
8
|
+
Language,
|
|
9
|
+
Match,
|
|
10
|
+
SeverityLevel,
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"CheckProfanityResult",
|
|
15
|
+
"ContextAwareConfig",
|
|
16
|
+
"FilterConfig",
|
|
17
|
+
"FilteredProfanityResult",
|
|
18
|
+
"Language",
|
|
19
|
+
"Match",
|
|
20
|
+
"SeverityLevel",
|
|
21
|
+
]
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Type definitions for glin-profanity Python package.
|
|
3
|
+
Unified API that mirrors the JavaScript/TypeScript package structure.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from enum import IntEnum
|
|
7
|
+
from typing import Literal, TypedDict
|
|
8
|
+
|
|
9
|
+
# Supported languages - unified list with JavaScript
|
|
10
|
+
Language = Literal[
|
|
11
|
+
"arabic",
|
|
12
|
+
"chinese",
|
|
13
|
+
"czech",
|
|
14
|
+
"danish",
|
|
15
|
+
"english",
|
|
16
|
+
"esperanto",
|
|
17
|
+
"finnish",
|
|
18
|
+
"french",
|
|
19
|
+
"german",
|
|
20
|
+
"hindi",
|
|
21
|
+
"hungarian",
|
|
22
|
+
"italian",
|
|
23
|
+
"japanese",
|
|
24
|
+
"korean",
|
|
25
|
+
"norwegian",
|
|
26
|
+
"persian",
|
|
27
|
+
"polish",
|
|
28
|
+
"portuguese",
|
|
29
|
+
"russian",
|
|
30
|
+
"spanish",
|
|
31
|
+
"swedish",
|
|
32
|
+
"thai",
|
|
33
|
+
"turkish",
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class SeverityLevel(IntEnum):
|
|
38
|
+
"""Severity levels for profanity matches - unified with JavaScript."""
|
|
39
|
+
|
|
40
|
+
EXACT = 1
|
|
41
|
+
FUZZY = 2
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class Match(TypedDict, total=False):
|
|
45
|
+
"""Represents a profanity match in text - unified with JavaScript."""
|
|
46
|
+
|
|
47
|
+
word: str
|
|
48
|
+
index: int
|
|
49
|
+
severity: SeverityLevel
|
|
50
|
+
context_score: float | None
|
|
51
|
+
reason: str | None
|
|
52
|
+
is_whitelisted: bool | None
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class CheckProfanityResult(TypedDict, total=False):
|
|
56
|
+
"""Result of profanity check operation - unified field names."""
|
|
57
|
+
|
|
58
|
+
contains_profanity: bool
|
|
59
|
+
profane_words: list[str]
|
|
60
|
+
processed_text: str | None
|
|
61
|
+
severity_map: dict[str, SeverityLevel] | None
|
|
62
|
+
matches: list[Match] | None
|
|
63
|
+
context_score: float | None
|
|
64
|
+
reason: str | None
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class ContextAwareConfig(TypedDict, total=False):
|
|
68
|
+
"""Configuration for context-aware filtering - unified with JavaScript."""
|
|
69
|
+
|
|
70
|
+
enable_context_aware: bool
|
|
71
|
+
context_window: int
|
|
72
|
+
confidence_threshold: float
|
|
73
|
+
domain_whitelists: dict[str, list[str]] | None
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class FilterConfig(ContextAwareConfig, total=False):
|
|
77
|
+
"""Main filter configuration options - unified with JavaScript."""
|
|
78
|
+
|
|
79
|
+
languages: list[Language] | None
|
|
80
|
+
all_languages: bool
|
|
81
|
+
case_sensitive: bool
|
|
82
|
+
word_boundaries: bool
|
|
83
|
+
custom_words: list[str] | None
|
|
84
|
+
replace_with: str | None
|
|
85
|
+
severity_levels: bool
|
|
86
|
+
ignore_words: list[str] | None
|
|
87
|
+
log_profanity: bool
|
|
88
|
+
allow_obfuscated_match: bool
|
|
89
|
+
fuzzy_tolerance_level: float
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class FilteredProfanityResult(TypedDict):
|
|
93
|
+
"""Result with minimum severity filtering."""
|
|
94
|
+
|
|
95
|
+
result: CheckProfanityResult
|
|
96
|
+
filtered_words: list[str]
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "glin-profanity"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Glin-Profanity is a lightweight and efficient Python package designed to detect and filter profane language in text inputs across multiple languages."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
authors = [
|
|
12
|
+
{ name = "glinr", email = "contact@glincker.com" },
|
|
13
|
+
]
|
|
14
|
+
maintainers = [
|
|
15
|
+
{ name = "glinr", email = "contact@glincker.com" },
|
|
16
|
+
]
|
|
17
|
+
keywords = [
|
|
18
|
+
"profanity",
|
|
19
|
+
"filter",
|
|
20
|
+
"moderation",
|
|
21
|
+
"content",
|
|
22
|
+
"text",
|
|
23
|
+
"nlp",
|
|
24
|
+
"language",
|
|
25
|
+
"detection",
|
|
26
|
+
"censorship",
|
|
27
|
+
"chat",
|
|
28
|
+
"comment",
|
|
29
|
+
"social",
|
|
30
|
+
"glin",
|
|
31
|
+
"glincker",
|
|
32
|
+
]
|
|
33
|
+
classifiers = [
|
|
34
|
+
"Development Status :: 5 - Production/Stable",
|
|
35
|
+
"Intended Audience :: Developers",
|
|
36
|
+
"License :: OSI Approved :: MIT License",
|
|
37
|
+
"Operating System :: OS Independent",
|
|
38
|
+
"Programming Language :: Python :: 3",
|
|
39
|
+
"Programming Language :: Python :: 3.10",
|
|
40
|
+
"Programming Language :: Python :: 3.11",
|
|
41
|
+
"Programming Language :: Python :: 3.12",
|
|
42
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
43
|
+
"Topic :: Text Processing :: Filters",
|
|
44
|
+
"Topic :: Internet :: WWW/HTTP :: Dynamic Content",
|
|
45
|
+
]
|
|
46
|
+
requires-python = ">=3.10"
|
|
47
|
+
dependencies = [
|
|
48
|
+
"typing-extensions>=4.0.0",
|
|
49
|
+
]
|
|
50
|
+
|
|
51
|
+
[project.optional-dependencies]
|
|
52
|
+
dev = [
|
|
53
|
+
"pytest>=7.0.0",
|
|
54
|
+
"pytest-cov>=4.0.0",
|
|
55
|
+
"black>=23.0.0",
|
|
56
|
+
"isort>=5.0.0",
|
|
57
|
+
"mypy>=1.0.0",
|
|
58
|
+
"ruff>=0.1.0",
|
|
59
|
+
]
|
|
60
|
+
test = [
|
|
61
|
+
"pytest>=7.0.0",
|
|
62
|
+
"pytest-cov>=4.0.0",
|
|
63
|
+
]
|
|
64
|
+
|
|
65
|
+
[project.urls]
|
|
66
|
+
Homepage = "https://www.glincker.com/tools/glin-profanity"
|
|
67
|
+
Documentation = "https://github.com/GLINCKER/glin-profanity"
|
|
68
|
+
Repository = "https://github.com/GLINCKER/glin-profanity"
|
|
69
|
+
Issues = "https://github.com/GLINCKER/glin-profanity/issues"
|
|
70
|
+
|
|
71
|
+
[tool.hatch.version]
|
|
72
|
+
path = "glin_profanity/__init__.py"
|
|
73
|
+
|
|
74
|
+
[tool.hatch.build.targets.wheel]
|
|
75
|
+
packages = ["glin_profanity"]
|
|
76
|
+
|
|
77
|
+
[tool.hatch.build.targets.sdist]
|
|
78
|
+
include = [
|
|
79
|
+
"/glin_profanity",
|
|
80
|
+
"/README.md",
|
|
81
|
+
]
|
|
82
|
+
exclude = [
|
|
83
|
+
"/tests",
|
|
84
|
+
"*.pyc",
|
|
85
|
+
"/__pycache__",
|
|
86
|
+
"/.pytest_cache",
|
|
87
|
+
"/.coverage",
|
|
88
|
+
"/htmlcov",
|
|
89
|
+
]
|
|
90
|
+
|
|
91
|
+
[tool.pytest.ini_options]
|
|
92
|
+
testpaths = ["tests"]
|
|
93
|
+
python_files = "test_*.py"
|
|
94
|
+
python_functions = "test_*"
|
|
95
|
+
addopts = "--verbose --cov=glin_profanity --cov-report=term-missing --cov-report=xml"
|
|
96
|
+
|
|
97
|
+
[tool.black]
|
|
98
|
+
line-length = 88
|
|
99
|
+
target-version = ["py310"]
|
|
100
|
+
include = '\.pyi?$'
|
|
101
|
+
|
|
102
|
+
[tool.isort]
|
|
103
|
+
profile = "black"
|
|
104
|
+
multi_line_output = 3
|
|
105
|
+
line_length = 88
|
|
106
|
+
|
|
107
|
+
[tool.mypy]
|
|
108
|
+
python_version = "3.10"
|
|
109
|
+
warn_return_any = true
|
|
110
|
+
warn_unused_configs = true
|
|
111
|
+
disallow_untyped_defs = true
|
|
112
|
+
disallow_incomplete_defs = true
|
|
113
|
+
check_untyped_defs = true
|
|
114
|
+
disallow_untyped_decorators = true
|
|
115
|
+
no_implicit_optional = true
|
|
116
|
+
warn_redundant_casts = true
|
|
117
|
+
warn_unused_ignores = true
|
|
118
|
+
warn_no_return = true
|
|
119
|
+
warn_unreachable = true
|
|
120
|
+
strict_equality = true
|
|
121
|
+
|
|
122
|
+
[tool.ruff]
|
|
123
|
+
line-length = 88
|
|
124
|
+
target-version = "py310"
|
|
125
|
+
|
|
126
|
+
[tool.ruff.lint]
|
|
127
|
+
# Focus on essential rules only - errors, imports, and basic style
|
|
128
|
+
select = [
|
|
129
|
+
"E", # pycodestyle errors
|
|
130
|
+
"F", # Pyflakes
|
|
131
|
+
"I", # isort
|
|
132
|
+
"W291", # trailing whitespace (keep this one)
|
|
133
|
+
"W292", # no newline at end of file
|
|
134
|
+
]
|
|
135
|
+
# Ignore specific rules that are too pedantic for CI
|
|
136
|
+
ignore = [
|
|
137
|
+
"W293", # blank line contains whitespace - too pedantic
|
|
138
|
+
]
|
|
139
|
+
|
|
140
|
+
[tool.hatch.envs.default]
|
|
141
|
+
dependencies = [
|
|
142
|
+
"pytest>=7.0.0",
|
|
143
|
+
"pytest-cov>=4.0.0",
|
|
144
|
+
"black>=23.0.0",
|
|
145
|
+
"isort>=5.0.0",
|
|
146
|
+
"mypy>=1.0.0",
|
|
147
|
+
"ruff>=0.1.0",
|
|
148
|
+
]
|
|
149
|
+
|
|
150
|
+
[tool.hatch.envs.default.scripts]
|
|
151
|
+
test = "pytest {args:tests}"
|
|
152
|
+
test-cov = "coverage run -m pytest {args:tests}"
|
|
153
|
+
cov-report = ["- coverage combine", "coverage report"]
|
|
154
|
+
cov = ["test-cov", "cov-report"]
|
|
155
|
+
|
|
156
|
+
[tool.ruff.lint.per-file-ignores]
|
|
157
|
+
"tests/**/*.py" = ["S101", "PLR2004", "ANN201"]
|