winidjango 2.0.11__py3-none-any.whl → 2.0.14__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of winidjango might be problematic. Click here for more details.
- winidjango-2.0.14.dist-info/METADATA +233 -0
- {winidjango-2.0.11.dist-info → winidjango-2.0.14.dist-info}/RECORD +4 -11
- winidjango/dev/tests/fixtures/fixture.py +0 -7
- winidjango/dev/tests/fixtures/scopes/__init__.py +0 -1
- winidjango/dev/tests/fixtures/scopes/class_.py +0 -8
- winidjango/dev/tests/fixtures/scopes/function.py +0 -8
- winidjango/dev/tests/fixtures/scopes/module.py +0 -8
- winidjango/dev/tests/fixtures/scopes/package.py +0 -8
- winidjango/dev/tests/fixtures/scopes/session.py +0 -8
- winidjango-2.0.11.dist-info/METADATA +0 -303
- {winidjango-2.0.11.dist-info → winidjango-2.0.14.dist-info}/WHEEL +0 -0
- {winidjango-2.0.11.dist-info → winidjango-2.0.14.dist-info}/entry_points.txt +0 -0
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: winidjango
|
|
3
|
+
Version: 2.0.14
|
|
4
|
+
Summary: A utils package for django
|
|
5
|
+
Author: Winipedia
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Requires-Dist: django
|
|
8
|
+
Requires-Dist: django-stubs-ext
|
|
9
|
+
Requires-Dist: winiutils
|
|
10
|
+
Requires-Python: >=3.12
|
|
11
|
+
Description-Content-Type: text/markdown
|
|
12
|
+
|
|
13
|
+
# winidjango
|
|
14
|
+
|
|
15
|
+
[](https://github.com/Winipedia/pyrig)
|
|
16
|
+
[](https://pypi.org/project/winidjango/)
|
|
17
|
+
[](https://www.python.org/downloads/)
|
|
18
|
+
[](https://www.djangoproject.com/)
|
|
19
|
+
[](https://opensource.org/licenses/MIT)
|
|
20
|
+
|
|
21
|
+
A production-ready Django utilities library that simplifies complex database operations and provides structured patterns for data management tasks.
|
|
22
|
+
|
|
23
|
+
## Table of Contents
|
|
24
|
+
|
|
25
|
+
- [Features](#features)
|
|
26
|
+
- [Installation](#installation)
|
|
27
|
+
- [Quick Start](#quick-start)
|
|
28
|
+
- [Documentation](#documentation)
|
|
29
|
+
- [Requirements](#requirements)
|
|
30
|
+
- [Development](#development)
|
|
31
|
+
- [Testing](#testing)
|
|
32
|
+
- [Contributing](#contributing)
|
|
33
|
+
- [License](#license)
|
|
34
|
+
|
|
35
|
+
## Features
|
|
36
|
+
|
|
37
|
+
### 🚀 High-Performance Bulk Operations
|
|
38
|
+
- **Multithreaded Processing**: Parallel execution of database operations for maximum speed
|
|
39
|
+
- **Automatic Chunking**: Configurable batch sizes (default: 1000) for memory-efficient processing
|
|
40
|
+
- **Transaction Safety**: Atomic operations with intelligent transaction management
|
|
41
|
+
- **Dependency Resolution**: Automatic topological sorting for foreign key relationships
|
|
42
|
+
|
|
43
|
+
### 🛠️ Database Utilities
|
|
44
|
+
- **Bulk Create/Update/Delete**: Process thousands of records efficiently
|
|
45
|
+
- **Deletion Simulation**: Preview cascade effects before executing destructive operations
|
|
46
|
+
- **Bulk Comparison**: Detect differences between datasets with field-level hashing
|
|
47
|
+
- **Raw SQL Execution**: Safe parameter binding with automatic cursor management
|
|
48
|
+
|
|
49
|
+
### 📦 Model Utilities
|
|
50
|
+
- **BaseModel**: Abstract base with `created_at`, `updated_at`, and type-safe `meta` property
|
|
51
|
+
- **Topological Sorting**: Automatic dependency ordering for model operations
|
|
52
|
+
- **Field Introspection**: Type-safe utilities for working with model fields
|
|
53
|
+
|
|
54
|
+
### 🎯 Management Command Framework
|
|
55
|
+
- **ABCBaseCommand**: Template method pattern with automatic logging
|
|
56
|
+
- **ImportDataBaseCommand**: Structured data import with Polars integration
|
|
57
|
+
- **Built-in Arguments**: Standard options for dry-run, batch size, threading, and more
|
|
58
|
+
- **Type Safety**: Full type hints with abstract method enforcement
|
|
59
|
+
|
|
60
|
+
## Installation
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
pip install winidjango
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
Or using `uv`:
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
uv add winidjango
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
## Quick Start
|
|
73
|
+
|
|
74
|
+
### Bulk Operations
|
|
75
|
+
|
|
76
|
+
```python
|
|
77
|
+
from winidjango.src.db.bulk import bulk_create_in_steps
|
|
78
|
+
|
|
79
|
+
# Create 10,000 records in batches of 1000
|
|
80
|
+
authors = [Author(name=f"Author {i}") for i in range(10000)]
|
|
81
|
+
created = bulk_create_in_steps(Author, authors, step=1000)
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
### Automatic Dependency Resolution
|
|
85
|
+
|
|
86
|
+
```python
|
|
87
|
+
from winidjango.src.db.bulk import bulk_create_bulks_in_steps
|
|
88
|
+
|
|
89
|
+
# Create related models in correct order automatically
|
|
90
|
+
results = bulk_create_bulks_in_steps({
|
|
91
|
+
Author: authors,
|
|
92
|
+
Book: books, # Created after Author
|
|
93
|
+
Review: reviews, # Created after Book
|
|
94
|
+
})
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
### Deletion Simulation
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
from winidjango.src.db.bulk import simulate_bulk_deletion
|
|
101
|
+
|
|
102
|
+
# Preview what would be deleted
|
|
103
|
+
deletion_preview = simulate_bulk_deletion(Author, authors_to_delete)
|
|
104
|
+
print(f"Would delete {len(deletion_preview[Author])} authors")
|
|
105
|
+
print(f"Would cascade delete {len(deletion_preview[Book])} books")
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
### Custom Management Command
|
|
109
|
+
|
|
110
|
+
```python
|
|
111
|
+
from winidjango.src.commands.base.base import ABCBaseCommand
|
|
112
|
+
from argparse import ArgumentParser
|
|
113
|
+
|
|
114
|
+
class MyCommand(ABCBaseCommand):
|
|
115
|
+
def add_command_arguments(self, parser: ArgumentParser) -> None:
|
|
116
|
+
parser.add_argument('--input-file', type=str, required=True)
|
|
117
|
+
|
|
118
|
+
def handle_command(self) -> None:
|
|
119
|
+
input_file = self.get_option('input_file')
|
|
120
|
+
dry_run = self.get_option('dry_run') # Built-in
|
|
121
|
+
|
|
122
|
+
if dry_run:
|
|
123
|
+
self.stdout.write('Dry run mode')
|
|
124
|
+
|
|
125
|
+
# Your logic here
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
### Data Import Command
|
|
129
|
+
|
|
130
|
+
```python
|
|
131
|
+
from winidjango.src.commands.import_data import ImportDataBaseCommand
|
|
132
|
+
import polars as pl
|
|
133
|
+
|
|
134
|
+
class ImportUsersCommand(ImportDataBaseCommand):
|
|
135
|
+
def handle_import(self) -> pl.DataFrame:
|
|
136
|
+
return pl.read_csv("users.csv")
|
|
137
|
+
|
|
138
|
+
def get_cleaning_df_cls(self) -> type[CleaningDF]:
|
|
139
|
+
return MyCleaningDF
|
|
140
|
+
|
|
141
|
+
def get_bulks_by_model(self, df: pl.DataFrame) -> dict[type[Model], Iterable[Model]]:
|
|
142
|
+
users = [User(name=row["name"]) for row in df.iter_rows(named=True)]
|
|
143
|
+
return {User: users}
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
## Documentation
|
|
147
|
+
|
|
148
|
+
Comprehensive documentation is available in the [`docs/`](docs/) directory:
|
|
149
|
+
|
|
150
|
+
- **[Database Utilities](docs/db.md)** - Bulk operations, model utilities, and SQL helpers
|
|
151
|
+
- **[Management Commands](docs/commands.md)** - Command framework and data import patterns
|
|
152
|
+
- **[API Reference](docs/index.md)** - Complete API documentation
|
|
153
|
+
|
|
154
|
+
## Requirements
|
|
155
|
+
|
|
156
|
+
- **Python**: 3.12+
|
|
157
|
+
- **Django**: Compatible with modern Django versions
|
|
158
|
+
- **Dependencies**:
|
|
159
|
+
- `django`
|
|
160
|
+
- `django-stubs-ext`
|
|
161
|
+
- `winiutils`
|
|
162
|
+
|
|
163
|
+
## Development
|
|
164
|
+
|
|
165
|
+
### Setup
|
|
166
|
+
|
|
167
|
+
```bash
|
|
168
|
+
# Clone the repository
|
|
169
|
+
git clone https://github.com/Winipedia/winidjango.git
|
|
170
|
+
cd winidjango
|
|
171
|
+
|
|
172
|
+
# Install dependencies
|
|
173
|
+
uv sync
|
|
174
|
+
|
|
175
|
+
# Install pre-commit hooks
|
|
176
|
+
pre-commit install
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
### Code Quality
|
|
180
|
+
|
|
181
|
+
This project uses:
|
|
182
|
+
- **mypy**: Strict type checking
|
|
183
|
+
- **ruff**: Linting and formatting
|
|
184
|
+
- **bandit**: Security analysis
|
|
185
|
+
- **pytest**: Testing framework
|
|
186
|
+
|
|
187
|
+
```bash
|
|
188
|
+
# Run type checking
|
|
189
|
+
mypy .
|
|
190
|
+
|
|
191
|
+
# Run linting
|
|
192
|
+
ruff check .
|
|
193
|
+
|
|
194
|
+
# Run security checks
|
|
195
|
+
bandit -r winidjango
|
|
196
|
+
|
|
197
|
+
# Format code
|
|
198
|
+
ruff format .
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
## Testing
|
|
202
|
+
|
|
203
|
+
```bash
|
|
204
|
+
# Run all tests
|
|
205
|
+
pytest
|
|
206
|
+
|
|
207
|
+
# Run with coverage
|
|
208
|
+
pytest --cov=winidjango
|
|
209
|
+
|
|
210
|
+
# Run specific test file
|
|
211
|
+
pytest tests/test_winidjango/test_src/test_db/test_bulk.py
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
## Contributing
|
|
215
|
+
|
|
216
|
+
Contributions are welcome! Please feel free to submit a Pull Request. For major changes, please open an issue first to discuss what you would like to change.
|
|
217
|
+
|
|
218
|
+
1. Fork the repository
|
|
219
|
+
2. Create your feature branch (`git checkout -b feature/amazing-feature`)
|
|
220
|
+
3. Commit your changes (`git commit -m 'Add some amazing feature'`)
|
|
221
|
+
4. Push to the branch (`git push origin feature/amazing-feature`)
|
|
222
|
+
5. Open a Pull Request
|
|
223
|
+
|
|
224
|
+
## License
|
|
225
|
+
|
|
226
|
+
This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
|
|
227
|
+
|
|
228
|
+
## Acknowledgments
|
|
229
|
+
|
|
230
|
+
- Built with [pyrig](https://github.com/Winipedia/pyrig) - Python project scaffolding tool
|
|
231
|
+
- Integrates with [winiutils](https://github.com/Winipedia/winiutils) - General Python utilities
|
|
232
|
+
|
|
233
|
+
---
|
|
@@ -9,13 +9,6 @@ winidjango/dev/configs/__init__.py,sha256=XHsbmjiaGom-KX-S3leCY9cJD3aP9p_0X6xYMc
|
|
|
9
9
|
winidjango/dev/configs/configs.py,sha256=N85kVaRHCHXc-ny0jbuYQwTcrxQJx_X_BcG30IcyrGw,586
|
|
10
10
|
winidjango/dev/tests/__init__.py,sha256=XHsbmjiaGom-KX-S3leCY9cJD3aP9p_0X6xYMcdkHBU,23
|
|
11
11
|
winidjango/dev/tests/fixtures/__init__.py,sha256=XHsbmjiaGom-KX-S3leCY9cJD3aP9p_0X6xYMcdkHBU,23
|
|
12
|
-
winidjango/dev/tests/fixtures/fixture.py,sha256=fA8GaETQ0foL1CEqvmxf6ab2Wz-_CxxI-ttkj8CUmok,263
|
|
13
|
-
winidjango/dev/tests/fixtures/scopes/__init__.py,sha256=XHsbmjiaGom-KX-S3leCY9cJD3aP9p_0X6xYMcdkHBU,23
|
|
14
|
-
winidjango/dev/tests/fixtures/scopes/class_.py,sha256=ZuI8_LEGy8SiutoygpgQq5_ojjlJRP-fNxDuhEzLj7M,417
|
|
15
|
-
winidjango/dev/tests/fixtures/scopes/function.py,sha256=2ehbrDhtw581pKmvY0KvuDkPCma8E19TyZboOb80L2I,428
|
|
16
|
-
winidjango/dev/tests/fixtures/scopes/module.py,sha256=OwdimjLzNMnsFT2th3OW5ZKP2YzjelC2NJ6jhSPrMMo,420
|
|
17
|
-
winidjango/dev/tests/fixtures/scopes/package.py,sha256=yTDD_7UcWdml7uahvIQ4XQKVeVl-TqAAFtbe9t9VG9c,424
|
|
18
|
-
winidjango/dev/tests/fixtures/scopes/session.py,sha256=MSN62vUCdpSPWBl9wYrEWu4qAmqwyINscgat2Rocp9M,423
|
|
19
12
|
winidjango/main.py,sha256=-w752ghb91kXeuIO1rvOiN_nGypBpQqLCJxVqMBP82c,409
|
|
20
13
|
winidjango/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
21
14
|
winidjango/src/__init__.py,sha256=KghXjKc_HAjcpkCULdXY9dN3SK2GIiYNPa3V5bCDVkE,19
|
|
@@ -28,7 +21,7 @@ winidjango/src/db/bulk.py,sha256=Rpw8pUK-swUMm2OrzR5XUzs4Gfx_OB0LvKQus13i9js,224
|
|
|
28
21
|
winidjango/src/db/fields.py,sha256=o_gJlb4D7FmNh9smL8qv0SbEeISjd4WXTu4fDQHNJC8,3616
|
|
29
22
|
winidjango/src/db/models.py,sha256=dV6ZdayNFfUCHB6Gpsp_V4a1ZiciKzsQXZ83oW_gj-Q,5052
|
|
30
23
|
winidjango/src/db/sql.py,sha256=MG9iTUTJCetCKGf75k-EyacUzbb-G4u_-NL9NTngFrk,2446
|
|
31
|
-
winidjango-2.0.
|
|
32
|
-
winidjango-2.0.
|
|
33
|
-
winidjango-2.0.
|
|
34
|
-
winidjango-2.0.
|
|
24
|
+
winidjango-2.0.14.dist-info/WHEEL,sha256=z-mOpxbJHqy3cq6SvUThBZdaLGFZzdZPtgWLcP2NKjQ,79
|
|
25
|
+
winidjango-2.0.14.dist-info/entry_points.txt,sha256=qM1ENsRWSLulhF8Cy92TeHW-4v8SAgzkRW7hNHXHG-4,55
|
|
26
|
+
winidjango-2.0.14.dist-info/METADATA,sha256=pvUNBGXkzrYW_AVx9bAq-F3ESPewJl8LvNPHnAEbREM,6762
|
|
27
|
+
winidjango-2.0.14.dist-info/RECORD,,
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
"""Fixtures for testing.
|
|
2
|
-
|
|
3
|
-
This module provides custom fixtures for pytest that can be pluued into tests
|
|
4
|
-
across the entire test suite.
|
|
5
|
-
All fixtures defined under the fixtures package are auto plugged in automatically
|
|
6
|
-
by pyrig via the pytest_plugins mechanism.
|
|
7
|
-
"""
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
"""__init__ module."""
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
"""Class-level test fixtures and utilities.
|
|
2
|
-
|
|
3
|
-
These fixtures in this module are automatically applied to all test classes
|
|
4
|
-
through pytest's autouse mechanism. Pyrig automatically adds this module to
|
|
5
|
-
pytest_plugins in conftest.py. However you still have decorate the fixture
|
|
6
|
-
with @autouse_class_fixture from pyrig.src.testing.fixtures or with pytest's
|
|
7
|
-
autouse mechanism @pytest.fixture(scope="class", autouse=True).
|
|
8
|
-
"""
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
"""Function-level test fixtures and utilities.
|
|
2
|
-
|
|
3
|
-
These fixtures in this module are automatically applied to all test functions
|
|
4
|
-
through pytest's autouse mechanism. Pyrig automatically adds this module to
|
|
5
|
-
pytest_plugins in conftest.py. However you still have decorate the fixture
|
|
6
|
-
with @autouse_function_fixture from pyrig.src.testing.fixtures or with pytest's
|
|
7
|
-
autouse mechanism @pytest.fixture(scope="function", autouse=True).
|
|
8
|
-
"""
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
"""Module-level test fixtures and utilities.
|
|
2
|
-
|
|
3
|
-
These fixtures in this module are automatically applied to all test modules
|
|
4
|
-
through pytest's autouse mechanism. Pyrig automatically adds this module to
|
|
5
|
-
pytest_plugins in conftest.py. However you still have decorate the fixture
|
|
6
|
-
with @autouse_module_fixture from pyrig.src.testing.fixtures or with pytest's
|
|
7
|
-
autouse mechanism @pytest.fixture(scope="module", autouse=True).
|
|
8
|
-
"""
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
"""Package-level test fixtures and utilities.
|
|
2
|
-
|
|
3
|
-
These fixtures in this module are automatically applied to all test packages
|
|
4
|
-
through pytest's autouse mechanism. Pyrig automatically adds this module to
|
|
5
|
-
pytest_plugins in conftest.py. However you still have decorate the fixture
|
|
6
|
-
with @autouse_package_fixture from pyrig.src.testing.fixtures or with pytest's
|
|
7
|
-
autouse mechanism @pytest.fixture(scope="package", autouse=True).
|
|
8
|
-
"""
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
"""Session-level test fixtures and utilities.
|
|
2
|
-
|
|
3
|
-
These fixtures in this module are automatically applied to the test session
|
|
4
|
-
through pytest's autouse mechanism. Pyrig automatically adds this module to
|
|
5
|
-
pytest_plugins in conftest.py. However you still have decorate the fixture
|
|
6
|
-
with @autouse_session_fixture from pyrig.src.testing.fixtures or with pytest's
|
|
7
|
-
autouse mechanism @pytest.fixture(scope="session", autouse=True).
|
|
8
|
-
"""
|
|
@@ -1,303 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: winidjango
|
|
3
|
-
Version: 2.0.11
|
|
4
|
-
Summary: A utils package for django
|
|
5
|
-
Author: Winipedia
|
|
6
|
-
Author-email: Winipedia <win.steveker@gmx.de>
|
|
7
|
-
License-Expression: MIT
|
|
8
|
-
Requires-Dist: django
|
|
9
|
-
Requires-Dist: django-stubs-ext
|
|
10
|
-
Requires-Dist: winiutils
|
|
11
|
-
Requires-Python: >=3.12
|
|
12
|
-
Description-Content-Type: text/markdown
|
|
13
|
-
|
|
14
|
-
# winidjango
|
|
15
|
-
|
|
16
|
-
(This project uses [pyrig](https://github.com/Winipedia/pyrig))
|
|
17
|
-
|
|
18
|
-
## Overview
|
|
19
|
-
|
|
20
|
-
**winidjango** is a production-ready Django utilities library that simplifies complex database operations and provides structured patterns for data management tasks. Built with type safety and performance in mind, it leverages modern Python features and integrates seamlessly with Django's ORM.
|
|
21
|
-
|
|
22
|
-
## Features
|
|
23
|
-
|
|
24
|
-
### Database Utilities (`winidjango.src.db`)
|
|
25
|
-
|
|
26
|
-
High-performance database operations with automatic optimization, dependency management, and type safety.
|
|
27
|
-
|
|
28
|
-
#### Bulk Operations (`bulk.py`)
|
|
29
|
-
|
|
30
|
-
Efficiently process large datasets with automatic chunking, multithreading, and transaction management.
|
|
31
|
-
|
|
32
|
-
**Core Functions:**
|
|
33
|
-
|
|
34
|
-
**`bulk_create_in_steps(model, bulk, step=1000)`**
|
|
35
|
-
- Creates thousands of model instances in configurable batches (default: 1000)
|
|
36
|
-
- Uses multithreading for parallel processing across chunks
|
|
37
|
-
- Returns list of created instances with populated PKs
|
|
38
|
-
- Wrapped in atomic transactions for data integrity
|
|
39
|
-
|
|
40
|
-
**`bulk_update_in_steps(model, bulk, update_fields, step=1000)`**
|
|
41
|
-
- Updates large datasets efficiently in batches
|
|
42
|
-
- Requires explicit `update_fields` list for safety
|
|
43
|
-
- Returns total count of updated objects
|
|
44
|
-
- Multithreaded processing for maximum performance
|
|
45
|
-
|
|
46
|
-
**`bulk_delete_in_steps(model, bulk, step=1000)`**
|
|
47
|
-
- Deletes in batches with cascade tracking
|
|
48
|
-
- Returns tuple: `(total_count, {model_name: count})`
|
|
49
|
-
- Tracks all cascade deletions across related models
|
|
50
|
-
- Safe handling of foreign key constraints
|
|
51
|
-
|
|
52
|
-
**`bulk_create_bulks_in_steps(bulk_by_class, step=1000)`**
|
|
53
|
-
- **Automatic Dependency Resolution**: Creates multiple model types in correct order
|
|
54
|
-
- Uses topological sorting to handle foreign key relationships
|
|
55
|
-
- Accepts dict mapping model classes to instance lists
|
|
56
|
-
- Returns dict with created instances (PKs populated)
|
|
57
|
-
|
|
58
|
-
**Advanced Comparison & Simulation:**
|
|
59
|
-
|
|
60
|
-
**`get_differences_between_bulks(bulk1, bulk2, fields)`**
|
|
61
|
-
- Compares two bulks by hashing field values
|
|
62
|
-
- Returns 4-tuple: `(in_1_not_2, in_2_not_1, in_both_from_1, in_both_from_2)`
|
|
63
|
-
- Useful for sync operations and change detection
|
|
64
|
-
- Preserves original object references
|
|
65
|
-
|
|
66
|
-
**`simulate_bulk_deletion(model_class, entries)`**
|
|
67
|
-
- **Preview deletions without executing** using Django's Collector
|
|
68
|
-
- Returns dict mapping models to objects that would be deleted
|
|
69
|
-
- Includes all cascade deletions
|
|
70
|
-
- Perfect for "what-if" analysis before destructive operations
|
|
71
|
-
|
|
72
|
-
**`multi_simulate_bulk_deletion(entries)`**
|
|
73
|
-
- Simulates deletions across multiple model types
|
|
74
|
-
- Aggregates cascade effects into single summary
|
|
75
|
-
- Accepts dict of `{model_class: [instances]}`
|
|
76
|
-
|
|
77
|
-
**Usage Examples:**
|
|
78
|
-
|
|
79
|
-
```python
|
|
80
|
-
from winidjango.src.db.bulk import (
|
|
81
|
-
bulk_create_in_steps,
|
|
82
|
-
bulk_create_bulks_in_steps,
|
|
83
|
-
get_differences_between_bulks,
|
|
84
|
-
simulate_bulk_deletion,
|
|
85
|
-
)
|
|
86
|
-
|
|
87
|
-
# Create 10,000 objects in batches of 1000
|
|
88
|
-
authors = [Author(name=f"Author {i}") for i in range(10000)]
|
|
89
|
-
created = bulk_create_in_steps(Author, authors, step=1000)
|
|
90
|
-
# Uses multithreading: ~10x faster than individual saves
|
|
91
|
-
|
|
92
|
-
# Create related models in dependency order
|
|
93
|
-
books = [Book(title=f"Book {i}", author=author) for i, author in enumerate(created)]
|
|
94
|
-
reviews = [Review(book=book, rating=5) for book in books]
|
|
95
|
-
|
|
96
|
-
results = bulk_create_bulks_in_steps({
|
|
97
|
-
Author: authors,
|
|
98
|
-
Book: books, # Created after Author (foreign key dependency)
|
|
99
|
-
Review: reviews, # Created after Book (foreign key dependency)
|
|
100
|
-
})
|
|
101
|
-
# Automatically sorted: Author → Book → Review
|
|
102
|
-
|
|
103
|
-
# Compare two datasets
|
|
104
|
-
from winidjango.src.db.fields import get_fields
|
|
105
|
-
fields = get_fields(Author)
|
|
106
|
-
old_authors = Author.objects.all()
|
|
107
|
-
new_authors = [Author(name=f"Updated {i}") for i in range(100)]
|
|
108
|
-
|
|
109
|
-
to_delete, to_create, unchanged_old, unchanged_new = get_differences_between_bulks(
|
|
110
|
-
list(old_authors), new_authors, fields
|
|
111
|
-
)
|
|
112
|
-
|
|
113
|
-
# Preview deletion impact
|
|
114
|
-
deletion_preview = simulate_bulk_deletion(Author, to_delete)
|
|
115
|
-
# Returns: {Author: {<Author: 1>, <Author: 2>}, Book: {<Book: 1>, <Book: 2>}, ...}
|
|
116
|
-
print(f"Would delete {len(deletion_preview[Author])} authors")
|
|
117
|
-
print(f"Would cascade delete {len(deletion_preview[Book])} books")
|
|
118
|
-
```
|
|
119
|
-
|
|
120
|
-
**Key Features:**
|
|
121
|
-
- **Multithreading**: Parallel processing of chunks for maximum speed
|
|
122
|
-
- **Transaction Safety**: Atomic operations with nested transaction warnings
|
|
123
|
-
- **Configurable Batch Size**: Default 1000, adjustable per operation
|
|
124
|
-
- **Type-Safe**: Full generic type hints with overloads
|
|
125
|
-
- **Memory Efficient**: Processes data in chunks, not all at once
|
|
126
|
-
|
|
127
|
-
#### Model Utilities (`models.py`)
|
|
128
|
-
|
|
129
|
-
**`topological_sort_models(models)`**
|
|
130
|
-
- Sorts models by foreign key dependencies using Python's `graphlib.TopologicalSorter`
|
|
131
|
-
- Ensures correct creation/deletion order
|
|
132
|
-
- Ignores self-referential relationships
|
|
133
|
-
- Raises `CycleError` for circular dependencies
|
|
134
|
-
|
|
135
|
-
**`hash_model_instance(instance, fields)`**
|
|
136
|
-
- Hashes model instances for comparison
|
|
137
|
-
- PK-based for saved instances (fast)
|
|
138
|
-
- Field-based for unsaved instances (content comparison)
|
|
139
|
-
- Used internally by `get_differences_between_bulks()`
|
|
140
|
-
|
|
141
|
-
**`BaseModel`** - Abstract base model with common fields:
|
|
142
|
-
- `created_at` - Auto-populated on creation
|
|
143
|
-
- `updated_at` - Auto-updated on save
|
|
144
|
-
- `meta` property - Type-safe access to `_meta`
|
|
145
|
-
- Custom `__str__()` and `__repr__()`
|
|
146
|
-
|
|
147
|
-
```python
|
|
148
|
-
from winidjango.src.db.models import BaseModel
|
|
149
|
-
|
|
150
|
-
class MyModel(BaseModel):
|
|
151
|
-
name = models.CharField(max_length=100)
|
|
152
|
-
|
|
153
|
-
class Meta(BaseModel.Meta):
|
|
154
|
-
db_table = "my_model"
|
|
155
|
-
|
|
156
|
-
# Automatically includes created_at and updated_at
|
|
157
|
-
obj = MyModel.objects.create(name="test")
|
|
158
|
-
print(obj.created_at) # datetime
|
|
159
|
-
print(obj) # "MyModel(1)"
|
|
160
|
-
```
|
|
161
|
-
|
|
162
|
-
#### Field Utilities (`fields.py`)
|
|
163
|
-
|
|
164
|
-
**`get_fields(model)`** - Get all fields including relationships
|
|
165
|
-
**`get_field_names(fields)`** - Extract field names from field objects
|
|
166
|
-
**`get_model_meta(model)`** - Type-safe access to model `_meta`
|
|
167
|
-
|
|
168
|
-
```python
|
|
169
|
-
from winidjango.src.db.fields import get_fields, get_field_names
|
|
170
|
-
|
|
171
|
-
fields = get_fields(User)
|
|
172
|
-
field_names = get_field_names(fields)
|
|
173
|
-
# ['id', 'username', 'email', 'groups', 'user_permissions', ...]
|
|
174
|
-
```
|
|
175
|
-
|
|
176
|
-
#### SQL Utilities (`sql.py`)
|
|
177
|
-
|
|
178
|
-
**`execute_sql(sql, params=None)`**
|
|
179
|
-
- Execute raw SQL with safe parameter binding
|
|
180
|
-
- Returns tuple: `(column_names, rows)`
|
|
181
|
-
- Automatic cursor management
|
|
182
|
-
- Protection against SQL injection
|
|
183
|
-
|
|
184
|
-
```python
|
|
185
|
-
from winidjango.src.db.sql import execute_sql
|
|
186
|
-
|
|
187
|
-
columns, rows = execute_sql(
|
|
188
|
-
"SELECT id, username FROM auth_user WHERE is_active = %(active)s",
|
|
189
|
-
params={"active": True}
|
|
190
|
-
)
|
|
191
|
-
# columns: ['id', 'username']
|
|
192
|
-
# rows: [(1, 'admin'), (2, 'user'), ...]
|
|
193
|
-
```
|
|
194
|
-
|
|
195
|
-
### Management Commands (`winidjango.src.commands`)
|
|
196
|
-
|
|
197
|
-
A powerful framework for building Django management commands with built-in best practices, automatic logging, and standardized argument handling.
|
|
198
|
-
|
|
199
|
-
#### `ABCBaseCommand` - Base Command Framework
|
|
200
|
-
|
|
201
|
-
Abstract base class that provides a robust foundation for all Django management commands:
|
|
202
|
-
|
|
203
|
-
**Key Features:**
|
|
204
|
-
- **Template Method Pattern**: Enforces consistent command structure while allowing customization
|
|
205
|
-
- **Automatic Logging**: All method calls are logged with performance tracking via `ABCLoggingMixin`
|
|
206
|
-
- **Built-in Common Arguments**: Pre-configured standard options available to all commands:
|
|
207
|
-
- `--dry_run` - Preview changes without executing
|
|
208
|
-
- `--force` - Force execution of actions
|
|
209
|
-
- `--delete` - Enable deletion operations
|
|
210
|
-
- `--yes` - Auto-confirm all prompts
|
|
211
|
-
- `--timeout` - Set command timeout
|
|
212
|
-
- `--batch_size` - Configure batch processing size
|
|
213
|
-
- `--threads` - Control thread count for parallel processing
|
|
214
|
-
- `--processes` - Control process count for multiprocessing
|
|
215
|
-
- **Type-Safe**: Full type hints with abstract method enforcement at compile-time
|
|
216
|
-
- **Structured Execution Flow**: Separates common setup (`base_handle`) from command-specific logic (`handle_command`)
|
|
217
|
-
|
|
218
|
-
**Usage Pattern:**
|
|
219
|
-
```python
|
|
220
|
-
from winidjango.src.commands.base.base import ABCBaseCommand
|
|
221
|
-
from argparse import ArgumentParser
|
|
222
|
-
|
|
223
|
-
class MyCommand(ABCBaseCommand):
|
|
224
|
-
def add_command_arguments(self, parser: ArgumentParser) -> None:
|
|
225
|
-
"""Add command-specific arguments."""
|
|
226
|
-
parser.add_argument('--input-file', type=str, required=True)
|
|
227
|
-
parser.add_argument('--output-format', choices=['json', 'csv'])
|
|
228
|
-
|
|
229
|
-
def handle_command(self) -> None:
|
|
230
|
-
"""Execute command logic."""
|
|
231
|
-
input_file = self.get_option('input_file')
|
|
232
|
-
dry_run = self.get_option('dry_run') # Built-in argument
|
|
233
|
-
batch_size = self.get_option('batch_size') # Built-in argument
|
|
234
|
-
|
|
235
|
-
if dry_run:
|
|
236
|
-
self.stdout.write('Dry run mode - no changes will be made')
|
|
237
|
-
|
|
238
|
-
# Your command logic here
|
|
239
|
-
self.process_data(input_file, batch_size)
|
|
240
|
-
```
|
|
241
|
-
|
|
242
|
-
#### `ImportDataBaseCommand` - Data Import Framework
|
|
243
|
-
|
|
244
|
-
Specialized command for structured data import workflows with automatic cleaning and bulk creation:
|
|
245
|
-
|
|
246
|
-
**Workflow Steps:**
|
|
247
|
-
1. **Import** (`handle_import()`) - Fetch raw data from any source, returns Polars DataFrame
|
|
248
|
-
2. **Clean** (`get_cleaning_df_cls()`) - Define data cleaning logic using `winiutils.CleaningDF`
|
|
249
|
-
3. **Transform** (`get_bulks_by_model()`) - Convert cleaned DataFrame to Django model instances
|
|
250
|
-
4. **Load** (`import_to_db()`) - Bulk create with automatic dependency resolution via topological sorting
|
|
251
|
-
|
|
252
|
-
**Key Features:**
|
|
253
|
-
- **Polars Integration**: High-performance data processing with Polars DataFrames
|
|
254
|
-
- **Automatic Cleaning**: Leverages `winiutils.CleaningDF` for standardized data cleaning pipeline
|
|
255
|
-
- **Dependency-Aware**: Uses `bulk_create_bulks_in_steps()` to handle foreign key relationships automatically
|
|
256
|
-
- **Inherits All Base Features**: Gets all `ABCBaseCommand` functionality (logging, common args, etc.)
|
|
257
|
-
|
|
258
|
-
**Usage Pattern:**
|
|
259
|
-
```python
|
|
260
|
-
from winidjango.src.commands.import_data import ImportDataBaseCommand
|
|
261
|
-
from winiutils.src.data.dataframe.cleaning import CleaningDF
|
|
262
|
-
import polars as pl
|
|
263
|
-
|
|
264
|
-
class MyCleaningDF(CleaningDF):
|
|
265
|
-
"""Define your data cleaning rules."""
|
|
266
|
-
NAME_COL = "name"
|
|
267
|
-
EMAIL_COL = "email"
|
|
268
|
-
|
|
269
|
-
@classmethod
|
|
270
|
-
def get_rename_map(cls) -> dict[str, str]:
|
|
271
|
-
return {"name": "user_name", "email": "user_email"}
|
|
272
|
-
|
|
273
|
-
@classmethod
|
|
274
|
-
def get_col_dtype_map(cls) -> dict[str, type[pl.DataType]]:
|
|
275
|
-
return {cls.NAME_COL: pl.Utf8, cls.EMAIL_COL: pl.Utf8}
|
|
276
|
-
|
|
277
|
-
# ... other cleaning methods
|
|
278
|
-
|
|
279
|
-
class ImportUsersCommand(ImportDataBaseCommand):
|
|
280
|
-
def handle_import(self) -> pl.DataFrame:
|
|
281
|
-
"""Fetch data from source."""
|
|
282
|
-
return pl.read_csv("users.csv")
|
|
283
|
-
|
|
284
|
-
def get_cleaning_df_cls(self) -> type[CleaningDF]:
|
|
285
|
-
"""Return your cleaning class."""
|
|
286
|
-
return MyCleaningDF
|
|
287
|
-
|
|
288
|
-
def get_bulks_by_model(self, df: pl.DataFrame) -> dict[type[Model], Iterable[Model]]:
|
|
289
|
-
"""Convert cleaned data to model instances."""
|
|
290
|
-
users = [User(name=row["name"], email=row["email"])
|
|
291
|
-
for row in df.iter_rows(named=True)]
|
|
292
|
-
profiles = [Profile(user=user) for user in users]
|
|
293
|
-
|
|
294
|
-
# Automatically created in correct order (User before Profile)
|
|
295
|
-
return {User: users, Profile: profiles}
|
|
296
|
-
```
|
|
297
|
-
|
|
298
|
-
**Benefits:**
|
|
299
|
-
- **Standardized Import Process**: Consistent pattern across all data import commands
|
|
300
|
-
- **Separation of Concerns**: Import, cleaning, and transformation logic clearly separated
|
|
301
|
-
- **Automatic Optimization**: Bulk operations with multithreading and dependency resolution
|
|
302
|
-
- **Data Quality**: Built-in cleaning pipeline ensures data consistency
|
|
303
|
-
- **Testable**: Each step can be tested independently
|
|
File without changes
|
|
File without changes
|