winipedia-utils 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- winipedia_utils-0.1.0/LICENSE +21 -0
- winipedia_utils-0.1.0/PKG-INFO +350 -0
- winipedia_utils-0.1.0/README.md +330 -0
- winipedia_utils-0.1.0/pyproject.toml +64 -0
- winipedia_utils-0.1.0/winipedia_utils/__init__.py +1 -0
- winipedia_utils-0.1.0/winipedia_utils/concurrent/__init__.py +1 -0
- winipedia_utils-0.1.0/winipedia_utils/concurrent/concurrent.py +242 -0
- winipedia_utils-0.1.0/winipedia_utils/concurrent/multiprocessing.py +115 -0
- winipedia_utils-0.1.0/winipedia_utils/concurrent/multithreading.py +93 -0
- winipedia_utils-0.1.0/winipedia_utils/consts.py +22 -0
- winipedia_utils-0.1.0/winipedia_utils/data/__init__.py +1 -0
- winipedia_utils-0.1.0/winipedia_utils/data/dataframe.py +7 -0
- winipedia_utils-0.1.0/winipedia_utils/django/__init__.py +27 -0
- winipedia_utils-0.1.0/winipedia_utils/django/bulk.py +536 -0
- winipedia_utils-0.1.0/winipedia_utils/django/command.py +334 -0
- winipedia_utils-0.1.0/winipedia_utils/django/database.py +304 -0
- winipedia_utils-0.1.0/winipedia_utils/git/__init__.py +1 -0
- winipedia_utils-0.1.0/winipedia_utils/git/gitignore.py +80 -0
- winipedia_utils-0.1.0/winipedia_utils/git/pre_commit/__init__.py +1 -0
- winipedia_utils-0.1.0/winipedia_utils/git/pre_commit/config.py +60 -0
- winipedia_utils-0.1.0/winipedia_utils/git/pre_commit/hooks.py +109 -0
- winipedia_utils-0.1.0/winipedia_utils/git/pre_commit/run_hooks.py +49 -0
- winipedia_utils-0.1.0/winipedia_utils/iterating/__init__.py +1 -0
- winipedia_utils-0.1.0/winipedia_utils/iterating/iterate.py +29 -0
- winipedia_utils-0.1.0/winipedia_utils/logging/__init__.py +1 -0
- winipedia_utils-0.1.0/winipedia_utils/logging/ansi.py +6 -0
- winipedia_utils-0.1.0/winipedia_utils/logging/config.py +64 -0
- winipedia_utils-0.1.0/winipedia_utils/logging/logger.py +26 -0
- winipedia_utils-0.1.0/winipedia_utils/modules/__init__.py +1 -0
- winipedia_utils-0.1.0/winipedia_utils/modules/class_.py +76 -0
- winipedia_utils-0.1.0/winipedia_utils/modules/function.py +86 -0
- winipedia_utils-0.1.0/winipedia_utils/modules/module.py +361 -0
- winipedia_utils-0.1.0/winipedia_utils/modules/package.py +350 -0
- winipedia_utils-0.1.0/winipedia_utils/oop/__init__.py +1 -0
- winipedia_utils-0.1.0/winipedia_utils/oop/mixins/__init__.py +1 -0
- winipedia_utils-0.1.0/winipedia_utils/oop/mixins/meta.py +315 -0
- winipedia_utils-0.1.0/winipedia_utils/oop/mixins/mixin.py +28 -0
- winipedia_utils-0.1.0/winipedia_utils/os/__init__.py +1 -0
- winipedia_utils-0.1.0/winipedia_utils/os/os.py +61 -0
- winipedia_utils-0.1.0/winipedia_utils/projects/__init__.py +1 -0
- winipedia_utils-0.1.0/winipedia_utils/projects/poetry/__init__.py +1 -0
- winipedia_utils-0.1.0/winipedia_utils/projects/poetry/config.py +91 -0
- winipedia_utils-0.1.0/winipedia_utils/projects/poetry/poetry.py +30 -0
- winipedia_utils-0.1.0/winipedia_utils/setup.py +36 -0
- winipedia_utils-0.1.0/winipedia_utils/testing/__init__.py +1 -0
- winipedia_utils-0.1.0/winipedia_utils/testing/assertions.py +23 -0
- winipedia_utils-0.1.0/winipedia_utils/testing/convention.py +177 -0
- winipedia_utils-0.1.0/winipedia_utils/testing/create_tests.py +286 -0
- winipedia_utils-0.1.0/winipedia_utils/testing/fixtures.py +28 -0
- winipedia_utils-0.1.0/winipedia_utils/testing/tests/__init__.py +1 -0
- winipedia_utils-0.1.0/winipedia_utils/testing/tests/base/__init__.py +1 -0
- winipedia_utils-0.1.0/winipedia_utils/testing/tests/base/fixtures/__init__.py +1 -0
- winipedia_utils-0.1.0/winipedia_utils/testing/tests/base/fixtures/fixture.py +6 -0
- winipedia_utils-0.1.0/winipedia_utils/testing/tests/base/fixtures/scopes/__init__.py +1 -0
- winipedia_utils-0.1.0/winipedia_utils/testing/tests/base/fixtures/scopes/class_.py +33 -0
- winipedia_utils-0.1.0/winipedia_utils/testing/tests/base/fixtures/scopes/function.py +7 -0
- winipedia_utils-0.1.0/winipedia_utils/testing/tests/base/fixtures/scopes/module.py +31 -0
- winipedia_utils-0.1.0/winipedia_utils/testing/tests/base/fixtures/scopes/package.py +7 -0
- winipedia_utils-0.1.0/winipedia_utils/testing/tests/base/fixtures/scopes/session.py +224 -0
- winipedia_utils-0.1.0/winipedia_utils/testing/tests/base/utils/__init__.py +1 -0
- winipedia_utils-0.1.0/winipedia_utils/testing/tests/base/utils/utils.py +82 -0
- winipedia_utils-0.1.0/winipedia_utils/testing/tests/conftest.py +26 -0
- winipedia_utils-0.1.0/winipedia_utils/text/__init__.py +1 -0
- winipedia_utils-0.1.0/winipedia_utils/text/string.py +126 -0
@@ -0,0 +1,21 @@
|
|
1
|
+
MIT License
|
2
|
+
|
3
|
+
Copyright (c) 2025 Winipedia
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
13
|
+
copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
21
|
+
SOFTWARE.
|
@@ -0,0 +1,350 @@
|
|
1
|
+
Metadata-Version: 2.3
|
2
|
+
Name: winipedia-utils
|
3
|
+
Version: 0.1.0
|
4
|
+
Summary: A package with many utility functions
|
5
|
+
License: MIT
|
6
|
+
Author: Winipedia
|
7
|
+
Requires-Python: >=3.12
|
8
|
+
Classifier: License :: OSI Approved :: MIT License
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
10
|
+
Classifier: Programming Language :: Python :: 3.12
|
11
|
+
Classifier: Programming Language :: Python :: 3.13
|
12
|
+
Requires-Dist: defusedxml (>=0.7.1,<0.8.0)
|
13
|
+
Requires-Dist: django (>=5.2.1,<6.0.0)
|
14
|
+
Requires-Dist: pathspec (>=0.12.1,<0.13.0)
|
15
|
+
Requires-Dist: setuptools (>=80.3.1,<81.0.0)
|
16
|
+
Requires-Dist: tomlkit (>=0.13.2,<0.14.0)
|
17
|
+
Requires-Dist: tqdm (>=4.67.1,<5.0.0)
|
18
|
+
Description-Content-Type: text/markdown
|
19
|
+
|
20
|
+
# Winipedia Utils
|
21
|
+
|
22
|
+
[](https://www.python.org/downloads/)
|
23
|
+
[](https://python-poetry.org/)
|
24
|
+
[](https://github.com/astral-sh/ruff)
|
25
|
+
[](https://mypy-lang.org/)
|
26
|
+
[](https://bandit.readthedocs.io/)
|
27
|
+
|
28
|
+
A comprehensive Python utility ecosystem designed to scale into an all-in-one toolkit for Python development. Winipedia Utils provides battle-tested utilities for Django, dataframes, strings, concurrent processing, iterating, and any other tools needed for full Python projects, while enforcing clean code practices through automated tooling.
|
29
|
+
|
30
|
+
## ๐ฏ Core Purpose
|
31
|
+
|
32
|
+
Winipedia Utils serves as the **foundation for reducing repeated code** across Python projects while **enforcing clean code practices** through automated tooling. The project setup script automatically configures your development environment with industry best practices, ensuring consistent code quality across all your projects.
|
33
|
+
|
34
|
+
### Key Benefits
|
35
|
+
|
36
|
+
- **๐ Zero-configuration setup** - Complete development environment in one command
|
37
|
+
- **๐งช Automated test generation** - 100% test coverage scaffolding for your entire codebase
|
38
|
+
- **๐ Quality assurance** - Automated linting, type checking, and security scanning
|
39
|
+
- **๐ฆ Comprehensive utilities** - Growing collection of production-ready utility functions
|
40
|
+
- **๐ Consistent standards** - Enforced code style and testing patterns across projects
|
41
|
+
|
42
|
+
## ๐๏ธ Some Implementation
|
43
|
+
|
44
|
+
### Project Setup & Automation
|
45
|
+
- **Automated dependency installation** (ruff, mypy, pytest, bandit, pre-commit)
|
46
|
+
- **Pre-commit hook configuration** for code quality enforcement
|
47
|
+
- **pyproject.toml configuration** with optimal tool settings
|
48
|
+
- **Complete development environment setup** in a single command
|
49
|
+
|
50
|
+
### Core Utility Modules
|
51
|
+
|
52
|
+
#### ๐งช Testing Infrastructure (`testing/`)
|
53
|
+
- **Automated test file generation** for entire codebases
|
54
|
+
- **Test stub creation** for every function, class, and method
|
55
|
+
- **Mirror test structure** maintaining 1:1 correspondence with source code
|
56
|
+
- **Testing convention enforcement** with pytest integration
|
57
|
+
|
58
|
+
#### ๐ Module Introspection (`modules/`)
|
59
|
+
- **Package discovery and manipulation** utilities
|
60
|
+
- **Dynamic module creation and import** capabilities
|
61
|
+
- **Code analysis and extraction** tools
|
62
|
+
- **Python object introspection** and manipulation
|
63
|
+
|
64
|
+
#### โก Concurrent Processing (`concurrent/`)
|
65
|
+
- **Multiprocessing utilities** with automatic worker optimization
|
66
|
+
- **Multithreading support** for I/O-bound tasks
|
67
|
+
- **Timeout handling** and process management
|
68
|
+
- **Progress tracking** with tqdm integration
|
69
|
+
|
70
|
+
#### ๐ Django Utilities (`django/`)
|
71
|
+
- **Bulk operations** with multithreaded processing for create, update, delete
|
72
|
+
- **Advanced BaseCommand** with logging, validation, and common arguments
|
73
|
+
- **Database utilities** including model hashing and topological sorting
|
74
|
+
- **Model introspection** and dependency analysis tools
|
75
|
+
|
76
|
+
#### ๐ Data Processing (`data/`)
|
77
|
+
- **DataFrame utilities** for data manipulation and analysis
|
78
|
+
- **Data cleaning and transformation** operations
|
79
|
+
- **Aggregation and preprocessing** tools
|
80
|
+
|
81
|
+
#### ๐ Iterating Utilities (`iterating/`)
|
82
|
+
- **Iterable manipulation** with safe length operations
|
83
|
+
- **Generator utilities** and iteration helpers
|
84
|
+
- **Collection processing** tools
|
85
|
+
|
86
|
+
#### ๐ Text Processing (`text/`)
|
87
|
+
- **String manipulation** utilities
|
88
|
+
- **XML parsing** with security features
|
89
|
+
- **Input handling** with timeout support
|
90
|
+
- **Hash generation** and text truncation
|
91
|
+
|
92
|
+
#### ๐ง Development Tools
|
93
|
+
- **Git integration** (`git/`) - gitignore handling, pre-commit management
|
94
|
+
- **Logging configuration** (`logging/`) - standardized logging setup
|
95
|
+
- **OS utilities** (`os/`) - command finding, subprocess management
|
96
|
+
- **OOP enhancements** (`oop/mixins/`) - advanced metaclasses and mixins
|
97
|
+
- **Project configuration** (`projects/poetry/`) - Poetry integration and management
|
98
|
+
|
99
|
+
## ๐ Installation & Quick Start
|
100
|
+
|
101
|
+
### Prerequisites
|
102
|
+
- Python 3.12+
|
103
|
+
- Poetry (for dependency management)
|
104
|
+
|
105
|
+
### Installation
|
106
|
+
|
107
|
+
```bash
|
108
|
+
# Add to your project
|
109
|
+
poetry add winipedia-utils
|
110
|
+
|
111
|
+
# Run the setup script
|
112
|
+
poetry run python -m winipedia_utils.setup
|
113
|
+
```
|
114
|
+
|
115
|
+
### What Happens During Setup
|
116
|
+
|
117
|
+
The setup script automatically:
|
118
|
+
|
119
|
+
1. **Installs development dependencies**: ruff, mypy, pytest, bandit, pre-commit, and type stubs
|
120
|
+
2. **Configures pre-commit hooks**: Automated code quality checks on every commit
|
121
|
+
3. **Sets up pyproject.toml**: Optimal configurations for all development tools
|
122
|
+
4. **Generates comprehensive tests**: Creates test files for your entire codebase
|
123
|
+
5. **Runs initial quality checks**: Ensures everything is properly configured
|
124
|
+
|
125
|
+
```python
|
126
|
+
# winipedia_utils/setup.py
|
127
|
+
def _setup() -> None:
|
128
|
+
"""Set up the project."""
|
129
|
+
_install_dev_dependencies() # Install quality tools
|
130
|
+
_add_package_hook_to_pre_commit_config() # Configure pre-commit
|
131
|
+
_add_tool_configurations_to_pyproject_toml() # Setup tool configs
|
132
|
+
_run_all_hooks() # Generate tests & run checks
|
133
|
+
logger.info("Setup complete!")
|
134
|
+
```
|
135
|
+
|
136
|
+
## ๐ก Usage Examples
|
137
|
+
|
138
|
+
### Automated Test Generation
|
139
|
+
|
140
|
+
```python
|
141
|
+
from winipedia_utils.testing.create_tests import create_tests
|
142
|
+
|
143
|
+
# Automatically generate test files for your entire project
|
144
|
+
create_tests()
|
145
|
+
# Creates comprehensive test structure with stubs for every function/class
|
146
|
+
```
|
147
|
+
|
148
|
+
### Module Introspection
|
149
|
+
|
150
|
+
```python
|
151
|
+
from winipedia_utils.modules.package import get_scr_package, walk_package
|
152
|
+
from winipedia_utils.modules.function import get_all_functions_from_module
|
153
|
+
|
154
|
+
# Discover your main source package
|
155
|
+
src_package = get_scr_package()
|
156
|
+
|
157
|
+
# Walk through all modules in a package
|
158
|
+
for package, modules in walk_package(src_package):
|
159
|
+
for module in modules:
|
160
|
+
functions = get_all_functions_from_module(module)
|
161
|
+
print(f"Found {len(functions)} functions in {module.__name__}")
|
162
|
+
```
|
163
|
+
|
164
|
+
### Concurrent Processing
|
165
|
+
|
166
|
+
```python
|
167
|
+
from winipedia_utils.concurrent.multiprocessing import multiprocess_loop
|
168
|
+
from winipedia_utils.concurrent.multithreading import multithread_loop
|
169
|
+
|
170
|
+
# CPU-bound tasks with multiprocessing
|
171
|
+
def cpu_intensive_task(data):
|
172
|
+
return complex_calculation(data)
|
173
|
+
|
174
|
+
results = multiprocess_loop(
|
175
|
+
process_function=cpu_intensive_task,
|
176
|
+
process_args=[[item] for item in large_dataset]
|
177
|
+
)
|
178
|
+
|
179
|
+
# I/O-bound tasks with multithreading
|
180
|
+
def io_task(url):
|
181
|
+
return fetch_data(url)
|
182
|
+
|
183
|
+
results = multithread_loop(
|
184
|
+
process_function=io_task,
|
185
|
+
process_args=[[url] for url in urls]
|
186
|
+
)
|
187
|
+
```
|
188
|
+
|
189
|
+
### Django Bulk Operations
|
190
|
+
|
191
|
+
```python
|
192
|
+
from winipedia_utils.django.bulk import (
|
193
|
+
bulk_create_in_steps,
|
194
|
+
bulk_update_in_steps,
|
195
|
+
bulk_delete_in_steps
|
196
|
+
)
|
197
|
+
|
198
|
+
# Efficient bulk creation with multithreading
|
199
|
+
created_objects = bulk_create_in_steps(
|
200
|
+
model=MyModel,
|
201
|
+
bulk=[MyModel(name=f"item_{i}") for i in range(10000)],
|
202
|
+
step=1000
|
203
|
+
)
|
204
|
+
|
205
|
+
# Bulk update with field specification
|
206
|
+
updated_count = bulk_update_in_steps(
|
207
|
+
model=MyModel,
|
208
|
+
bulk=objects_to_update,
|
209
|
+
update_fields=['name', 'status'],
|
210
|
+
step=1000
|
211
|
+
)
|
212
|
+
|
213
|
+
# Safe bulk deletion with cascade handling
|
214
|
+
deleted_count, deletion_summary = bulk_delete_in_steps(
|
215
|
+
model=MyModel,
|
216
|
+
bulk=objects_to_delete,
|
217
|
+
step=1000
|
218
|
+
)
|
219
|
+
```
|
220
|
+
|
221
|
+
### Django Management Commands
|
222
|
+
|
223
|
+
```python
|
224
|
+
from winipedia_utils.django.command import ABCBaseCommand
|
225
|
+
|
226
|
+
class MyCommand(ABCBaseCommand):
|
227
|
+
"""Custom Django command with built-in logging and validation."""
|
228
|
+
|
229
|
+
help = "Process data with automatic logging and error handling"
|
230
|
+
|
231
|
+
def add_command_arguments(self, parser):
|
232
|
+
parser.add_argument('--batch-size', type=int, default=1000)
|
233
|
+
|
234
|
+
def handle_command(self, *args, **options):
|
235
|
+
# Command logic with automatic logging and performance tracking
|
236
|
+
batch_size = options['batch_size']
|
237
|
+
self.stdout.write(f"Processing with batch size: {batch_size}")
|
238
|
+
```
|
239
|
+
|
240
|
+
### String Utilities
|
241
|
+
|
242
|
+
```python
|
243
|
+
from winipedia_utils.text.string import (
|
244
|
+
value_to_truncated_string,
|
245
|
+
ask_for_input_with_timeout,
|
246
|
+
get_reusable_hash
|
247
|
+
)
|
248
|
+
|
249
|
+
# Safely truncate any value to string
|
250
|
+
truncated = value_to_truncated_string(large_object, max_length=100)
|
251
|
+
|
252
|
+
# Get user input with timeout
|
253
|
+
try:
|
254
|
+
user_input = ask_for_input_with_timeout("Enter value: ", timeout=30)
|
255
|
+
except TimeoutError:
|
256
|
+
print("Input timeout exceeded")
|
257
|
+
|
258
|
+
# Generate consistent hashes
|
259
|
+
hash_value = get_reusable_hash("some data")
|
260
|
+
```
|
261
|
+
|
262
|
+
### Iterating Utilities
|
263
|
+
|
264
|
+
```python
|
265
|
+
from winipedia_utils.iterating.iterate import get_len_with_default
|
266
|
+
|
267
|
+
# Safe length operations with fallback
|
268
|
+
length = get_len_with_default(some_iterable, default=0)
|
269
|
+
|
270
|
+
# Works with generators and other iterables that don't support len()
|
271
|
+
gen = (x for x in range(100))
|
272
|
+
safe_length = get_len_with_default(gen, default=100)
|
273
|
+
```
|
274
|
+
|
275
|
+
### Advanced OOP Features
|
276
|
+
|
277
|
+
```python
|
278
|
+
from winipedia_utils.oop.mixins.meta import ABCImplementationLoggingMeta
|
279
|
+
|
280
|
+
class MyClass(metaclass=ABCImplementationLoggingMeta):
|
281
|
+
"""Class with automatic logging and implementation enforcement."""
|
282
|
+
|
283
|
+
def my_method(self):
|
284
|
+
# Automatically logged with performance tracking
|
285
|
+
return "result"
|
286
|
+
```
|
287
|
+
|
288
|
+
## ๐ฎ Future Vision
|
289
|
+
|
290
|
+
Winipedia Utils is designed to scale into a comprehensive ecosystem covering many utilities
|
291
|
+
|
292
|
+
## ๐ก๏ธ Development Standards
|
293
|
+
|
294
|
+
### Automated Quality Assurance
|
295
|
+
|
296
|
+
Every utility in the ecosystem benefits from:
|
297
|
+
|
298
|
+
- **๐งช 100% Test Coverage Scaffolding**: Automated test generation ensures no function goes untested
|
299
|
+
- **๐ Static Type Checking**: MyPy ensures type safety across all utilities
|
300
|
+
- **๐จ Code Formatting**: Ruff enforces consistent code style
|
301
|
+
- **๐ Security Scanning**: Bandit identifies potential security issues
|
302
|
+
- **๐ Documentation Standards**: Consistent docstring patterns and examples
|
303
|
+
|
304
|
+
### Pre-commit Hooks
|
305
|
+
|
306
|
+
Automatically configured hooks ensure:
|
307
|
+
- Code formatting with ruff
|
308
|
+
- Type checking with mypy
|
309
|
+
- Security scanning with bandit
|
310
|
+
- Test generation and execution
|
311
|
+
- Dependency validation
|
312
|
+
See details in `winipedia_utils/git/pre_commit/hooks.py` and `winipedia_utils/git/pre_commit/run_hooks.py`
|
313
|
+
|
314
|
+
## ๐๏ธ Project Structure
|
315
|
+
|
316
|
+
```
|
317
|
+
winipedia_utils/
|
318
|
+
โโโ concurrent/ # Parallel processing utilities
|
319
|
+
โโโ conventions/ # Testing and naming conventions
|
320
|
+
โโโ data/ # Data science utilities (expanding)
|
321
|
+
โโโ git/ # Git integration and workflows
|
322
|
+
โโโ logging/ # Standardized logging configuration
|
323
|
+
โโโ modules/ # Package/module introspection
|
324
|
+
โโโ oop/ # Object-oriented programming enhancements
|
325
|
+
โโโ os/ # Operating system utilities
|
326
|
+
โโโ projects/ # Project configuration (Poetry, etc.)
|
327
|
+
โโโ testing/ # Automated test generation
|
328
|
+
โโโ text/ # String and text processing
|
329
|
+
โโโ setup.py # Main setup script
|
330
|
+
```
|
331
|
+
|
332
|
+
## ๐ค Contributing
|
333
|
+
|
334
|
+
Winipedia Utils welcomes contributions! The automated setup ensures that all contributions maintain high quality standards:
|
335
|
+
|
336
|
+
1. Fork the repository
|
337
|
+
2. Run `poetry run python -m winipedia_utils.setup` in your fork
|
338
|
+
3. Add your utilities following the established patterns
|
339
|
+
4. Tests are automatically generated - implement the test logic
|
340
|
+
5. Pre-commit hooks ensure code quality
|
341
|
+
6. Submit a pull request
|
342
|
+
|
343
|
+
## ๐ License
|
344
|
+
|
345
|
+
MIT License - see LICENSE file for details.
|
346
|
+
|
347
|
+
---
|
348
|
+
|
349
|
+
**Note**: All `_private_methods` are intended for internal use within the winipedia_utils package and cannot be used directly in external projects.
|
350
|
+
|
@@ -0,0 +1,330 @@
|
|
1
|
+
# Winipedia Utils
|
2
|
+
|
3
|
+
[](https://www.python.org/downloads/)
|
4
|
+
[](https://python-poetry.org/)
|
5
|
+
[](https://github.com/astral-sh/ruff)
|
6
|
+
[](https://mypy-lang.org/)
|
7
|
+
[](https://bandit.readthedocs.io/)
|
8
|
+
|
9
|
+
A comprehensive Python utility ecosystem designed to scale into an all-in-one toolkit for Python development. Winipedia Utils provides battle-tested utilities for Django, dataframes, strings, concurrent processing, iterating, and any other tools needed for full Python projects, while enforcing clean code practices through automated tooling.
|
10
|
+
|
11
|
+
## ๐ฏ Core Purpose
|
12
|
+
|
13
|
+
Winipedia Utils serves as the **foundation for reducing repeated code** across Python projects while **enforcing clean code practices** through automated tooling. The project setup script automatically configures your development environment with industry best practices, ensuring consistent code quality across all your projects.
|
14
|
+
|
15
|
+
### Key Benefits
|
16
|
+
|
17
|
+
- **๐ Zero-configuration setup** - Complete development environment in one command
|
18
|
+
- **๐งช Automated test generation** - 100% test coverage scaffolding for your entire codebase
|
19
|
+
- **๐ Quality assurance** - Automated linting, type checking, and security scanning
|
20
|
+
- **๐ฆ Comprehensive utilities** - Growing collection of production-ready utility functions
|
21
|
+
- **๐ Consistent standards** - Enforced code style and testing patterns across projects
|
22
|
+
|
23
|
+
## ๐๏ธ Some Implementation
|
24
|
+
|
25
|
+
### Project Setup & Automation
|
26
|
+
- **Automated dependency installation** (ruff, mypy, pytest, bandit, pre-commit)
|
27
|
+
- **Pre-commit hook configuration** for code quality enforcement
|
28
|
+
- **pyproject.toml configuration** with optimal tool settings
|
29
|
+
- **Complete development environment setup** in a single command
|
30
|
+
|
31
|
+
### Core Utility Modules
|
32
|
+
|
33
|
+
#### ๐งช Testing Infrastructure (`testing/`)
|
34
|
+
- **Automated test file generation** for entire codebases
|
35
|
+
- **Test stub creation** for every function, class, and method
|
36
|
+
- **Mirror test structure** maintaining 1:1 correspondence with source code
|
37
|
+
- **Testing convention enforcement** with pytest integration
|
38
|
+
|
39
|
+
#### ๐ Module Introspection (`modules/`)
|
40
|
+
- **Package discovery and manipulation** utilities
|
41
|
+
- **Dynamic module creation and import** capabilities
|
42
|
+
- **Code analysis and extraction** tools
|
43
|
+
- **Python object introspection** and manipulation
|
44
|
+
|
45
|
+
#### โก Concurrent Processing (`concurrent/`)
|
46
|
+
- **Multiprocessing utilities** with automatic worker optimization
|
47
|
+
- **Multithreading support** for I/O-bound tasks
|
48
|
+
- **Timeout handling** and process management
|
49
|
+
- **Progress tracking** with tqdm integration
|
50
|
+
|
51
|
+
#### ๐ Django Utilities (`django/`)
|
52
|
+
- **Bulk operations** with multithreaded processing for create, update, delete
|
53
|
+
- **Advanced BaseCommand** with logging, validation, and common arguments
|
54
|
+
- **Database utilities** including model hashing and topological sorting
|
55
|
+
- **Model introspection** and dependency analysis tools
|
56
|
+
|
57
|
+
#### ๐ Data Processing (`data/`)
|
58
|
+
- **DataFrame utilities** for data manipulation and analysis
|
59
|
+
- **Data cleaning and transformation** operations
|
60
|
+
- **Aggregation and preprocessing** tools
|
61
|
+
|
62
|
+
#### ๐ Iterating Utilities (`iterating/`)
|
63
|
+
- **Iterable manipulation** with safe length operations
|
64
|
+
- **Generator utilities** and iteration helpers
|
65
|
+
- **Collection processing** tools
|
66
|
+
|
67
|
+
#### ๐ Text Processing (`text/`)
|
68
|
+
- **String manipulation** utilities
|
69
|
+
- **XML parsing** with security features
|
70
|
+
- **Input handling** with timeout support
|
71
|
+
- **Hash generation** and text truncation
|
72
|
+
|
73
|
+
#### ๐ง Development Tools
|
74
|
+
- **Git integration** (`git/`) - gitignore handling, pre-commit management
|
75
|
+
- **Logging configuration** (`logging/`) - standardized logging setup
|
76
|
+
- **OS utilities** (`os/`) - command finding, subprocess management
|
77
|
+
- **OOP enhancements** (`oop/mixins/`) - advanced metaclasses and mixins
|
78
|
+
- **Project configuration** (`projects/poetry/`) - Poetry integration and management
|
79
|
+
|
80
|
+
## ๐ Installation & Quick Start
|
81
|
+
|
82
|
+
### Prerequisites
|
83
|
+
- Python 3.12+
|
84
|
+
- Poetry (for dependency management)
|
85
|
+
|
86
|
+
### Installation
|
87
|
+
|
88
|
+
```bash
|
89
|
+
# Add to your project
|
90
|
+
poetry add winipedia-utils
|
91
|
+
|
92
|
+
# Run the setup script
|
93
|
+
poetry run python -m winipedia_utils.setup
|
94
|
+
```
|
95
|
+
|
96
|
+
### What Happens During Setup
|
97
|
+
|
98
|
+
The setup script automatically:
|
99
|
+
|
100
|
+
1. **Installs development dependencies**: ruff, mypy, pytest, bandit, pre-commit, and type stubs
|
101
|
+
2. **Configures pre-commit hooks**: Automated code quality checks on every commit
|
102
|
+
3. **Sets up pyproject.toml**: Optimal configurations for all development tools
|
103
|
+
4. **Generates comprehensive tests**: Creates test files for your entire codebase
|
104
|
+
5. **Runs initial quality checks**: Ensures everything is properly configured
|
105
|
+
|
106
|
+
```python
|
107
|
+
# winipedia_utils/setup.py
|
108
|
+
def _setup() -> None:
|
109
|
+
"""Set up the project."""
|
110
|
+
_install_dev_dependencies() # Install quality tools
|
111
|
+
_add_package_hook_to_pre_commit_config() # Configure pre-commit
|
112
|
+
_add_tool_configurations_to_pyproject_toml() # Setup tool configs
|
113
|
+
_run_all_hooks() # Generate tests & run checks
|
114
|
+
logger.info("Setup complete!")
|
115
|
+
```
|
116
|
+
|
117
|
+
## ๐ก Usage Examples
|
118
|
+
|
119
|
+
### Automated Test Generation
|
120
|
+
|
121
|
+
```python
|
122
|
+
from winipedia_utils.testing.create_tests import create_tests
|
123
|
+
|
124
|
+
# Automatically generate test files for your entire project
|
125
|
+
create_tests()
|
126
|
+
# Creates comprehensive test structure with stubs for every function/class
|
127
|
+
```
|
128
|
+
|
129
|
+
### Module Introspection
|
130
|
+
|
131
|
+
```python
|
132
|
+
from winipedia_utils.modules.package import get_scr_package, walk_package
|
133
|
+
from winipedia_utils.modules.function import get_all_functions_from_module
|
134
|
+
|
135
|
+
# Discover your main source package
|
136
|
+
src_package = get_scr_package()
|
137
|
+
|
138
|
+
# Walk through all modules in a package
|
139
|
+
for package, modules in walk_package(src_package):
|
140
|
+
for module in modules:
|
141
|
+
functions = get_all_functions_from_module(module)
|
142
|
+
print(f"Found {len(functions)} functions in {module.__name__}")
|
143
|
+
```
|
144
|
+
|
145
|
+
### Concurrent Processing
|
146
|
+
|
147
|
+
```python
|
148
|
+
from winipedia_utils.concurrent.multiprocessing import multiprocess_loop
|
149
|
+
from winipedia_utils.concurrent.multithreading import multithread_loop
|
150
|
+
|
151
|
+
# CPU-bound tasks with multiprocessing
|
152
|
+
def cpu_intensive_task(data):
|
153
|
+
return complex_calculation(data)
|
154
|
+
|
155
|
+
results = multiprocess_loop(
|
156
|
+
process_function=cpu_intensive_task,
|
157
|
+
process_args=[[item] for item in large_dataset]
|
158
|
+
)
|
159
|
+
|
160
|
+
# I/O-bound tasks with multithreading
|
161
|
+
def io_task(url):
|
162
|
+
return fetch_data(url)
|
163
|
+
|
164
|
+
results = multithread_loop(
|
165
|
+
process_function=io_task,
|
166
|
+
process_args=[[url] for url in urls]
|
167
|
+
)
|
168
|
+
```
|
169
|
+
|
170
|
+
### Django Bulk Operations
|
171
|
+
|
172
|
+
```python
|
173
|
+
from winipedia_utils.django.bulk import (
|
174
|
+
bulk_create_in_steps,
|
175
|
+
bulk_update_in_steps,
|
176
|
+
bulk_delete_in_steps
|
177
|
+
)
|
178
|
+
|
179
|
+
# Efficient bulk creation with multithreading
|
180
|
+
created_objects = bulk_create_in_steps(
|
181
|
+
model=MyModel,
|
182
|
+
bulk=[MyModel(name=f"item_{i}") for i in range(10000)],
|
183
|
+
step=1000
|
184
|
+
)
|
185
|
+
|
186
|
+
# Bulk update with field specification
|
187
|
+
updated_count = bulk_update_in_steps(
|
188
|
+
model=MyModel,
|
189
|
+
bulk=objects_to_update,
|
190
|
+
update_fields=['name', 'status'],
|
191
|
+
step=1000
|
192
|
+
)
|
193
|
+
|
194
|
+
# Safe bulk deletion with cascade handling
|
195
|
+
deleted_count, deletion_summary = bulk_delete_in_steps(
|
196
|
+
model=MyModel,
|
197
|
+
bulk=objects_to_delete,
|
198
|
+
step=1000
|
199
|
+
)
|
200
|
+
```
|
201
|
+
|
202
|
+
### Django Management Commands
|
203
|
+
|
204
|
+
```python
|
205
|
+
from winipedia_utils.django.command import ABCBaseCommand
|
206
|
+
|
207
|
+
class MyCommand(ABCBaseCommand):
|
208
|
+
"""Custom Django command with built-in logging and validation."""
|
209
|
+
|
210
|
+
help = "Process data with automatic logging and error handling"
|
211
|
+
|
212
|
+
def add_command_arguments(self, parser):
|
213
|
+
parser.add_argument('--batch-size', type=int, default=1000)
|
214
|
+
|
215
|
+
def handle_command(self, *args, **options):
|
216
|
+
# Command logic with automatic logging and performance tracking
|
217
|
+
batch_size = options['batch_size']
|
218
|
+
self.stdout.write(f"Processing with batch size: {batch_size}")
|
219
|
+
```
|
220
|
+
|
221
|
+
### String Utilities
|
222
|
+
|
223
|
+
```python
|
224
|
+
from winipedia_utils.text.string import (
|
225
|
+
value_to_truncated_string,
|
226
|
+
ask_for_input_with_timeout,
|
227
|
+
get_reusable_hash
|
228
|
+
)
|
229
|
+
|
230
|
+
# Safely truncate any value to string
|
231
|
+
truncated = value_to_truncated_string(large_object, max_length=100)
|
232
|
+
|
233
|
+
# Get user input with timeout
|
234
|
+
try:
|
235
|
+
user_input = ask_for_input_with_timeout("Enter value: ", timeout=30)
|
236
|
+
except TimeoutError:
|
237
|
+
print("Input timeout exceeded")
|
238
|
+
|
239
|
+
# Generate consistent hashes
|
240
|
+
hash_value = get_reusable_hash("some data")
|
241
|
+
```
|
242
|
+
|
243
|
+
### Iterating Utilities
|
244
|
+
|
245
|
+
```python
|
246
|
+
from winipedia_utils.iterating.iterate import get_len_with_default
|
247
|
+
|
248
|
+
# Safe length operations with fallback
|
249
|
+
length = get_len_with_default(some_iterable, default=0)
|
250
|
+
|
251
|
+
# Works with generators and other iterables that don't support len()
|
252
|
+
gen = (x for x in range(100))
|
253
|
+
safe_length = get_len_with_default(gen, default=100)
|
254
|
+
```
|
255
|
+
|
256
|
+
### Advanced OOP Features
|
257
|
+
|
258
|
+
```python
|
259
|
+
from winipedia_utils.oop.mixins.meta import ABCImplementationLoggingMeta
|
260
|
+
|
261
|
+
class MyClass(metaclass=ABCImplementationLoggingMeta):
|
262
|
+
"""Class with automatic logging and implementation enforcement."""
|
263
|
+
|
264
|
+
def my_method(self):
|
265
|
+
# Automatically logged with performance tracking
|
266
|
+
return "result"
|
267
|
+
```
|
268
|
+
|
269
|
+
## ๐ฎ Future Vision
|
270
|
+
|
271
|
+
Winipedia Utils is designed to scale into a comprehensive ecosystem covering many utilities
|
272
|
+
|
273
|
+
## ๐ก๏ธ Development Standards
|
274
|
+
|
275
|
+
### Automated Quality Assurance
|
276
|
+
|
277
|
+
Every utility in the ecosystem benefits from:
|
278
|
+
|
279
|
+
- **๐งช 100% Test Coverage Scaffolding**: Automated test generation ensures no function goes untested
|
280
|
+
- **๐ Static Type Checking**: MyPy ensures type safety across all utilities
|
281
|
+
- **๐จ Code Formatting**: Ruff enforces consistent code style
|
282
|
+
- **๐ Security Scanning**: Bandit identifies potential security issues
|
283
|
+
- **๐ Documentation Standards**: Consistent docstring patterns and examples
|
284
|
+
|
285
|
+
### Pre-commit Hooks
|
286
|
+
|
287
|
+
Automatically configured hooks ensure:
|
288
|
+
- Code formatting with ruff
|
289
|
+
- Type checking with mypy
|
290
|
+
- Security scanning with bandit
|
291
|
+
- Test generation and execution
|
292
|
+
- Dependency validation
|
293
|
+
See details in `winipedia_utils/git/pre_commit/hooks.py` and `winipedia_utils/git/pre_commit/run_hooks.py`
|
294
|
+
|
295
|
+
## ๐๏ธ Project Structure
|
296
|
+
|
297
|
+
```
|
298
|
+
winipedia_utils/
|
299
|
+
โโโ concurrent/ # Parallel processing utilities
|
300
|
+
โโโ conventions/ # Testing and naming conventions
|
301
|
+
โโโ data/ # Data science utilities (expanding)
|
302
|
+
โโโ git/ # Git integration and workflows
|
303
|
+
โโโ logging/ # Standardized logging configuration
|
304
|
+
โโโ modules/ # Package/module introspection
|
305
|
+
โโโ oop/ # Object-oriented programming enhancements
|
306
|
+
โโโ os/ # Operating system utilities
|
307
|
+
โโโ projects/ # Project configuration (Poetry, etc.)
|
308
|
+
โโโ testing/ # Automated test generation
|
309
|
+
โโโ text/ # String and text processing
|
310
|
+
โโโ setup.py # Main setup script
|
311
|
+
```
|
312
|
+
|
313
|
+
## ๐ค Contributing
|
314
|
+
|
315
|
+
Winipedia Utils welcomes contributions! The automated setup ensures that all contributions maintain high quality standards:
|
316
|
+
|
317
|
+
1. Fork the repository
|
318
|
+
2. Run `poetry run python -m winipedia_utils.setup` in your fork
|
319
|
+
3. Add your utilities following the established patterns
|
320
|
+
4. Tests are automatically generated - implement the test logic
|
321
|
+
5. Pre-commit hooks ensure code quality
|
322
|
+
6. Submit a pull request
|
323
|
+
|
324
|
+
## ๐ License
|
325
|
+
|
326
|
+
MIT License - see LICENSE file for details.
|
327
|
+
|
328
|
+
---
|
329
|
+
|
330
|
+
**Note**: All `_private_methods` are intended for internal use within the winipedia_utils package and cannot be used directly in external projects.
|