winidjango 2.0.12__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- winidjango-2.0.12/PKG-INFO +303 -0
- winidjango-2.0.12/README.md +290 -0
- winidjango-2.0.12/pyproject.toml +95 -0
- winidjango-2.0.12/winidjango/__init__.py +34 -0
- winidjango-2.0.12/winidjango/dev/__init__.py +1 -0
- winidjango-2.0.12/winidjango/dev/artifacts/__init__.py +1 -0
- winidjango-2.0.12/winidjango/dev/artifacts/builders/__init__.py +1 -0
- winidjango-2.0.12/winidjango/dev/artifacts/resources/__init__.py +1 -0
- winidjango-2.0.12/winidjango/dev/cli/__init__.py +1 -0
- winidjango-2.0.12/winidjango/dev/cli/subcommands.py +6 -0
- winidjango-2.0.12/winidjango/dev/configs/__init__.py +1 -0
- winidjango-2.0.12/winidjango/dev/configs/configs.py +18 -0
- winidjango-2.0.12/winidjango/dev/tests/__init__.py +1 -0
- winidjango-2.0.12/winidjango/dev/tests/fixtures/__init__.py +1 -0
- winidjango-2.0.12/winidjango/dev/tests/fixtures/fixture.py +7 -0
- winidjango-2.0.12/winidjango/dev/tests/fixtures/scopes/__init__.py +1 -0
- winidjango-2.0.12/winidjango/dev/tests/fixtures/scopes/class_.py +8 -0
- winidjango-2.0.12/winidjango/dev/tests/fixtures/scopes/function.py +8 -0
- winidjango-2.0.12/winidjango/dev/tests/fixtures/scopes/module.py +8 -0
- winidjango-2.0.12/winidjango/dev/tests/fixtures/scopes/package.py +8 -0
- winidjango-2.0.12/winidjango/dev/tests/fixtures/scopes/session.py +8 -0
- winidjango-2.0.12/winidjango/main.py +19 -0
- winidjango-2.0.12/winidjango/py.typed +0 -0
- winidjango-2.0.12/winidjango/src/__init__.py +1 -0
- winidjango-2.0.12/winidjango/src/commands/__init__.py +1 -0
- winidjango-2.0.12/winidjango/src/commands/base/__init__.py +1 -0
- winidjango-2.0.12/winidjango/src/commands/base/base.py +305 -0
- winidjango-2.0.12/winidjango/src/commands/import_data.py +76 -0
- winidjango-2.0.12/winidjango/src/db/__init__.py +1 -0
- winidjango-2.0.12/winidjango/src/db/bulk.py +644 -0
- winidjango-2.0.12/winidjango/src/db/fields.py +101 -0
- winidjango-2.0.12/winidjango/src/db/models.py +145 -0
- winidjango-2.0.12/winidjango/src/db/sql.py +63 -0
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: winidjango
|
|
3
|
+
Version: 2.0.12
|
|
4
|
+
Summary: A utils package for django
|
|
5
|
+
Author: Winipedia
|
|
6
|
+
Author-email: Winipedia <win.steveker@gmx.de>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
Requires-Dist: django
|
|
9
|
+
Requires-Dist: django-stubs-ext
|
|
10
|
+
Requires-Dist: winiutils
|
|
11
|
+
Requires-Python: >=3.12
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
|
|
14
|
+
# winidjango
|
|
15
|
+
|
|
16
|
+
(This project uses [pyrig](https://github.com/Winipedia/pyrig))
|
|
17
|
+
|
|
18
|
+
## Overview
|
|
19
|
+
|
|
20
|
+
**winidjango** is a production-ready Django utilities library that simplifies complex database operations and provides structured patterns for data management tasks. Built with type safety and performance in mind, it leverages modern Python features and integrates seamlessly with Django's ORM.
|
|
21
|
+
|
|
22
|
+
## Features
|
|
23
|
+
|
|
24
|
+
### Database Utilities (`winidjango.src.db`)
|
|
25
|
+
|
|
26
|
+
High-performance database operations with automatic optimization, dependency management, and type safety.
|
|
27
|
+
|
|
28
|
+
#### Bulk Operations (`bulk.py`)
|
|
29
|
+
|
|
30
|
+
Efficiently process large datasets with automatic chunking, multithreading, and transaction management.
|
|
31
|
+
|
|
32
|
+
**Core Functions:**
|
|
33
|
+
|
|
34
|
+
**`bulk_create_in_steps(model, bulk, step=1000)`**
|
|
35
|
+
- Creates thousands of model instances in configurable batches (default: 1000)
|
|
36
|
+
- Uses multithreading for parallel processing across chunks
|
|
37
|
+
- Returns list of created instances with populated PKs
|
|
38
|
+
- Wrapped in atomic transactions for data integrity
|
|
39
|
+
|
|
40
|
+
**`bulk_update_in_steps(model, bulk, update_fields, step=1000)`**
|
|
41
|
+
- Updates large datasets efficiently in batches
|
|
42
|
+
- Requires explicit `update_fields` list for safety
|
|
43
|
+
- Returns total count of updated objects
|
|
44
|
+
- Multithreaded processing for maximum performance
|
|
45
|
+
|
|
46
|
+
**`bulk_delete_in_steps(model, bulk, step=1000)`**
|
|
47
|
+
- Deletes in batches with cascade tracking
|
|
48
|
+
- Returns tuple: `(total_count, {model_name: count})`
|
|
49
|
+
- Tracks all cascade deletions across related models
|
|
50
|
+
- Safe handling of foreign key constraints
|
|
51
|
+
|
|
52
|
+
**`bulk_create_bulks_in_steps(bulk_by_class, step=1000)`**
|
|
53
|
+
- **Automatic Dependency Resolution**: Creates multiple model types in correct order
|
|
54
|
+
- Uses topological sorting to handle foreign key relationships
|
|
55
|
+
- Accepts dict mapping model classes to instance lists
|
|
56
|
+
- Returns dict with created instances (PKs populated)
|
|
57
|
+
|
|
58
|
+
**Advanced Comparison & Simulation:**
|
|
59
|
+
|
|
60
|
+
**`get_differences_between_bulks(bulk1, bulk2, fields)`**
|
|
61
|
+
- Compares two bulks by hashing field values
|
|
62
|
+
- Returns 4-tuple: `(in_1_not_2, in_2_not_1, in_both_from_1, in_both_from_2)`
|
|
63
|
+
- Useful for sync operations and change detection
|
|
64
|
+
- Preserves original object references
|
|
65
|
+
|
|
66
|
+
**`simulate_bulk_deletion(model_class, entries)`**
|
|
67
|
+
- **Preview deletions without executing** using Django's Collector
|
|
68
|
+
- Returns dict mapping models to objects that would be deleted
|
|
69
|
+
- Includes all cascade deletions
|
|
70
|
+
- Perfect for "what-if" analysis before destructive operations
|
|
71
|
+
|
|
72
|
+
**`multi_simulate_bulk_deletion(entries)`**
|
|
73
|
+
- Simulates deletions across multiple model types
|
|
74
|
+
- Aggregates cascade effects into single summary
|
|
75
|
+
- Accepts dict of `{model_class: [instances]}`
|
|
76
|
+
|
|
77
|
+
**Usage Examples:**
|
|
78
|
+
|
|
79
|
+
```python
|
|
80
|
+
from winidjango.src.db.bulk import (
|
|
81
|
+
bulk_create_in_steps,
|
|
82
|
+
bulk_create_bulks_in_steps,
|
|
83
|
+
get_differences_between_bulks,
|
|
84
|
+
simulate_bulk_deletion,
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
# Create 10,000 objects in batches of 1000
|
|
88
|
+
authors = [Author(name=f"Author {i}") for i in range(10000)]
|
|
89
|
+
created = bulk_create_in_steps(Author, authors, step=1000)
|
|
90
|
+
# Uses multithreading: ~10x faster than individual saves
|
|
91
|
+
|
|
92
|
+
# Create related models in dependency order
|
|
93
|
+
books = [Book(title=f"Book {i}", author=author) for i, author in enumerate(created)]
|
|
94
|
+
reviews = [Review(book=book, rating=5) for book in books]
|
|
95
|
+
|
|
96
|
+
results = bulk_create_bulks_in_steps({
|
|
97
|
+
Author: authors,
|
|
98
|
+
Book: books, # Created after Author (foreign key dependency)
|
|
99
|
+
Review: reviews, # Created after Book (foreign key dependency)
|
|
100
|
+
})
|
|
101
|
+
# Automatically sorted: Author → Book → Review
|
|
102
|
+
|
|
103
|
+
# Compare two datasets
|
|
104
|
+
from winidjango.src.db.fields import get_fields
|
|
105
|
+
fields = get_fields(Author)
|
|
106
|
+
old_authors = Author.objects.all()
|
|
107
|
+
new_authors = [Author(name=f"Updated {i}") for i in range(100)]
|
|
108
|
+
|
|
109
|
+
to_delete, to_create, unchanged_old, unchanged_new = get_differences_between_bulks(
|
|
110
|
+
list(old_authors), new_authors, fields
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
# Preview deletion impact
|
|
114
|
+
deletion_preview = simulate_bulk_deletion(Author, to_delete)
|
|
115
|
+
# Returns: {Author: {<Author: 1>, <Author: 2>}, Book: {<Book: 1>, <Book: 2>}, ...}
|
|
116
|
+
print(f"Would delete {len(deletion_preview[Author])} authors")
|
|
117
|
+
print(f"Would cascade delete {len(deletion_preview[Book])} books")
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
**Key Features:**
|
|
121
|
+
- **Multithreading**: Parallel processing of chunks for maximum speed
|
|
122
|
+
- **Transaction Safety**: Atomic operations with nested transaction warnings
|
|
123
|
+
- **Configurable Batch Size**: Default 1000, adjustable per operation
|
|
124
|
+
- **Type-Safe**: Full generic type hints with overloads
|
|
125
|
+
- **Memory Efficient**: Processes data in chunks, not all at once
|
|
126
|
+
|
|
127
|
+
#### Model Utilities (`models.py`)
|
|
128
|
+
|
|
129
|
+
**`topological_sort_models(models)`**
|
|
130
|
+
- Sorts models by foreign key dependencies using Python's `graphlib.TopologicalSorter`
|
|
131
|
+
- Ensures correct creation/deletion order
|
|
132
|
+
- Ignores self-referential relationships
|
|
133
|
+
- Raises `CycleError` for circular dependencies
|
|
134
|
+
|
|
135
|
+
**`hash_model_instance(instance, fields)`**
|
|
136
|
+
- Hashes model instances for comparison
|
|
137
|
+
- PK-based for saved instances (fast)
|
|
138
|
+
- Field-based for unsaved instances (content comparison)
|
|
139
|
+
- Used internally by `get_differences_between_bulks()`
|
|
140
|
+
|
|
141
|
+
**`BaseModel`** - Abstract base model with common fields:
|
|
142
|
+
- `created_at` - Auto-populated on creation
|
|
143
|
+
- `updated_at` - Auto-updated on save
|
|
144
|
+
- `meta` property - Type-safe access to `_meta`
|
|
145
|
+
- Custom `__str__()` and `__repr__()`
|
|
146
|
+
|
|
147
|
+
```python
|
|
148
|
+
from winidjango.src.db.models import BaseModel
|
|
149
|
+
|
|
150
|
+
class MyModel(BaseModel):
|
|
151
|
+
name = models.CharField(max_length=100)
|
|
152
|
+
|
|
153
|
+
class Meta(BaseModel.Meta):
|
|
154
|
+
db_table = "my_model"
|
|
155
|
+
|
|
156
|
+
# Automatically includes created_at and updated_at
|
|
157
|
+
obj = MyModel.objects.create(name="test")
|
|
158
|
+
print(obj.created_at) # datetime
|
|
159
|
+
print(obj) # "MyModel(1)"
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
#### Field Utilities (`fields.py`)
|
|
163
|
+
|
|
164
|
+
**`get_fields(model)`** - Get all fields including relationships
|
|
165
|
+
**`get_field_names(fields)`** - Extract field names from field objects
|
|
166
|
+
**`get_model_meta(model)`** - Type-safe access to model `_meta`
|
|
167
|
+
|
|
168
|
+
```python
|
|
169
|
+
from winidjango.src.db.fields import get_fields, get_field_names
|
|
170
|
+
|
|
171
|
+
fields = get_fields(User)
|
|
172
|
+
field_names = get_field_names(fields)
|
|
173
|
+
# ['id', 'username', 'email', 'groups', 'user_permissions', ...]
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
#### SQL Utilities (`sql.py`)
|
|
177
|
+
|
|
178
|
+
**`execute_sql(sql, params=None)`**
|
|
179
|
+
- Execute raw SQL with safe parameter binding
|
|
180
|
+
- Returns tuple: `(column_names, rows)`
|
|
181
|
+
- Automatic cursor management
|
|
182
|
+
- Protection against SQL injection
|
|
183
|
+
|
|
184
|
+
```python
|
|
185
|
+
from winidjango.src.db.sql import execute_sql
|
|
186
|
+
|
|
187
|
+
columns, rows = execute_sql(
|
|
188
|
+
"SELECT id, username FROM auth_user WHERE is_active = %(active)s",
|
|
189
|
+
params={"active": True}
|
|
190
|
+
)
|
|
191
|
+
# columns: ['id', 'username']
|
|
192
|
+
# rows: [(1, 'admin'), (2, 'user'), ...]
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
### Management Commands (`winidjango.src.commands`)
|
|
196
|
+
|
|
197
|
+
A powerful framework for building Django management commands with built-in best practices, automatic logging, and standardized argument handling.
|
|
198
|
+
|
|
199
|
+
#### `ABCBaseCommand` - Base Command Framework
|
|
200
|
+
|
|
201
|
+
Abstract base class that provides a robust foundation for all Django management commands:
|
|
202
|
+
|
|
203
|
+
**Key Features:**
|
|
204
|
+
- **Template Method Pattern**: Enforces consistent command structure while allowing customization
|
|
205
|
+
- **Automatic Logging**: All method calls are logged with performance tracking via `ABCLoggingMixin`
|
|
206
|
+
- **Built-in Common Arguments**: Pre-configured standard options available to all commands:
|
|
207
|
+
- `--dry_run` - Preview changes without executing
|
|
208
|
+
- `--force` - Force execution of actions
|
|
209
|
+
- `--delete` - Enable deletion operations
|
|
210
|
+
- `--yes` - Auto-confirm all prompts
|
|
211
|
+
- `--timeout` - Set command timeout
|
|
212
|
+
- `--batch_size` - Configure batch processing size
|
|
213
|
+
- `--threads` - Control thread count for parallel processing
|
|
214
|
+
- `--processes` - Control process count for multiprocessing
|
|
215
|
+
- **Type-Safe**: Full type hints with abstract method enforcement at compile-time
|
|
216
|
+
- **Structured Execution Flow**: Separates common setup (`base_handle`) from command-specific logic (`handle_command`)
|
|
217
|
+
|
|
218
|
+
**Usage Pattern:**
|
|
219
|
+
```python
|
|
220
|
+
from winidjango.src.commands.base.base import ABCBaseCommand
|
|
221
|
+
from argparse import ArgumentParser
|
|
222
|
+
|
|
223
|
+
class MyCommand(ABCBaseCommand):
|
|
224
|
+
def add_command_arguments(self, parser: ArgumentParser) -> None:
|
|
225
|
+
"""Add command-specific arguments."""
|
|
226
|
+
parser.add_argument('--input-file', type=str, required=True)
|
|
227
|
+
parser.add_argument('--output-format', choices=['json', 'csv'])
|
|
228
|
+
|
|
229
|
+
def handle_command(self) -> None:
|
|
230
|
+
"""Execute command logic."""
|
|
231
|
+
input_file = self.get_option('input_file')
|
|
232
|
+
dry_run = self.get_option('dry_run') # Built-in argument
|
|
233
|
+
batch_size = self.get_option('batch_size') # Built-in argument
|
|
234
|
+
|
|
235
|
+
if dry_run:
|
|
236
|
+
self.stdout.write('Dry run mode - no changes will be made')
|
|
237
|
+
|
|
238
|
+
# Your command logic here
|
|
239
|
+
self.process_data(input_file, batch_size)
|
|
240
|
+
```
|
|
241
|
+
|
|
242
|
+
#### `ImportDataBaseCommand` - Data Import Framework
|
|
243
|
+
|
|
244
|
+
Specialized command for structured data import workflows with automatic cleaning and bulk creation:
|
|
245
|
+
|
|
246
|
+
**Workflow Steps:**
|
|
247
|
+
1. **Import** (`handle_import()`) - Fetch raw data from any source, returns Polars DataFrame
|
|
248
|
+
2. **Clean** (`get_cleaning_df_cls()`) - Define data cleaning logic using `winiutils.CleaningDF`
|
|
249
|
+
3. **Transform** (`get_bulks_by_model()`) - Convert cleaned DataFrame to Django model instances
|
|
250
|
+
4. **Load** (`import_to_db()`) - Bulk create with automatic dependency resolution via topological sorting
|
|
251
|
+
|
|
252
|
+
**Key Features:**
|
|
253
|
+
- **Polars Integration**: High-performance data processing with Polars DataFrames
|
|
254
|
+
- **Automatic Cleaning**: Leverages `winiutils.CleaningDF` for standardized data cleaning pipeline
|
|
255
|
+
- **Dependency-Aware**: Uses `bulk_create_bulks_in_steps()` to handle foreign key relationships automatically
|
|
256
|
+
- **Inherits All Base Features**: Gets all `ABCBaseCommand` functionality (logging, common args, etc.)
|
|
257
|
+
|
|
258
|
+
**Usage Pattern:**
|
|
259
|
+
```python
|
|
260
|
+
from winidjango.src.commands.import_data import ImportDataBaseCommand
|
|
261
|
+
from winiutils.src.data.dataframe.cleaning import CleaningDF
|
|
262
|
+
import polars as pl
|
|
263
|
+
|
|
264
|
+
class MyCleaningDF(CleaningDF):
|
|
265
|
+
"""Define your data cleaning rules."""
|
|
266
|
+
NAME_COL = "name"
|
|
267
|
+
EMAIL_COL = "email"
|
|
268
|
+
|
|
269
|
+
@classmethod
|
|
270
|
+
def get_rename_map(cls) -> dict[str, str]:
|
|
271
|
+
return {"name": "user_name", "email": "user_email"}
|
|
272
|
+
|
|
273
|
+
@classmethod
|
|
274
|
+
def get_col_dtype_map(cls) -> dict[str, type[pl.DataType]]:
|
|
275
|
+
return {cls.NAME_COL: pl.Utf8, cls.EMAIL_COL: pl.Utf8}
|
|
276
|
+
|
|
277
|
+
# ... other cleaning methods
|
|
278
|
+
|
|
279
|
+
class ImportUsersCommand(ImportDataBaseCommand):
|
|
280
|
+
def handle_import(self) -> pl.DataFrame:
|
|
281
|
+
"""Fetch data from source."""
|
|
282
|
+
return pl.read_csv("users.csv")
|
|
283
|
+
|
|
284
|
+
def get_cleaning_df_cls(self) -> type[CleaningDF]:
|
|
285
|
+
"""Return your cleaning class."""
|
|
286
|
+
return MyCleaningDF
|
|
287
|
+
|
|
288
|
+
def get_bulks_by_model(self, df: pl.DataFrame) -> dict[type[Model], Iterable[Model]]:
|
|
289
|
+
"""Convert cleaned data to model instances."""
|
|
290
|
+
users = [User(name=row["name"], email=row["email"])
|
|
291
|
+
for row in df.iter_rows(named=True)]
|
|
292
|
+
profiles = [Profile(user=user) for user in users]
|
|
293
|
+
|
|
294
|
+
# Automatically created in correct order (User before Profile)
|
|
295
|
+
return {User: users, Profile: profiles}
|
|
296
|
+
```
|
|
297
|
+
|
|
298
|
+
**Benefits:**
|
|
299
|
+
- **Standardized Import Process**: Consistent pattern across all data import commands
|
|
300
|
+
- **Separation of Concerns**: Import, cleaning, and transformation logic clearly separated
|
|
301
|
+
- **Automatic Optimization**: Bulk operations with multithreading and dependency resolution
|
|
302
|
+
- **Data Quality**: Built-in cleaning pipeline ensures data consistency
|
|
303
|
+
- **Testable**: Each step can be tested independently
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
# winidjango
|
|
2
|
+
|
|
3
|
+
(This project uses [pyrig](https://github.com/Winipedia/pyrig))
|
|
4
|
+
|
|
5
|
+
## Overview
|
|
6
|
+
|
|
7
|
+
**winidjango** is a production-ready Django utilities library that simplifies complex database operations and provides structured patterns for data management tasks. Built with type safety and performance in mind, it leverages modern Python features and integrates seamlessly with Django's ORM.
|
|
8
|
+
|
|
9
|
+
## Features
|
|
10
|
+
|
|
11
|
+
### Database Utilities (`winidjango.src.db`)
|
|
12
|
+
|
|
13
|
+
High-performance database operations with automatic optimization, dependency management, and type safety.
|
|
14
|
+
|
|
15
|
+
#### Bulk Operations (`bulk.py`)
|
|
16
|
+
|
|
17
|
+
Efficiently process large datasets with automatic chunking, multithreading, and transaction management.
|
|
18
|
+
|
|
19
|
+
**Core Functions:**
|
|
20
|
+
|
|
21
|
+
**`bulk_create_in_steps(model, bulk, step=1000)`**
|
|
22
|
+
- Creates thousands of model instances in configurable batches (default: 1000)
|
|
23
|
+
- Uses multithreading for parallel processing across chunks
|
|
24
|
+
- Returns list of created instances with populated PKs
|
|
25
|
+
- Wrapped in atomic transactions for data integrity
|
|
26
|
+
|
|
27
|
+
**`bulk_update_in_steps(model, bulk, update_fields, step=1000)`**
|
|
28
|
+
- Updates large datasets efficiently in batches
|
|
29
|
+
- Requires explicit `update_fields` list for safety
|
|
30
|
+
- Returns total count of updated objects
|
|
31
|
+
- Multithreaded processing for maximum performance
|
|
32
|
+
|
|
33
|
+
**`bulk_delete_in_steps(model, bulk, step=1000)`**
|
|
34
|
+
- Deletes in batches with cascade tracking
|
|
35
|
+
- Returns tuple: `(total_count, {model_name: count})`
|
|
36
|
+
- Tracks all cascade deletions across related models
|
|
37
|
+
- Safe handling of foreign key constraints
|
|
38
|
+
|
|
39
|
+
**`bulk_create_bulks_in_steps(bulk_by_class, step=1000)`**
|
|
40
|
+
- **Automatic Dependency Resolution**: Creates multiple model types in correct order
|
|
41
|
+
- Uses topological sorting to handle foreign key relationships
|
|
42
|
+
- Accepts dict mapping model classes to instance lists
|
|
43
|
+
- Returns dict with created instances (PKs populated)
|
|
44
|
+
|
|
45
|
+
**Advanced Comparison & Simulation:**
|
|
46
|
+
|
|
47
|
+
**`get_differences_between_bulks(bulk1, bulk2, fields)`**
|
|
48
|
+
- Compares two bulks by hashing field values
|
|
49
|
+
- Returns 4-tuple: `(in_1_not_2, in_2_not_1, in_both_from_1, in_both_from_2)`
|
|
50
|
+
- Useful for sync operations and change detection
|
|
51
|
+
- Preserves original object references
|
|
52
|
+
|
|
53
|
+
**`simulate_bulk_deletion(model_class, entries)`**
|
|
54
|
+
- **Preview deletions without executing** using Django's Collector
|
|
55
|
+
- Returns dict mapping models to objects that would be deleted
|
|
56
|
+
- Includes all cascade deletions
|
|
57
|
+
- Perfect for "what-if" analysis before destructive operations
|
|
58
|
+
|
|
59
|
+
**`multi_simulate_bulk_deletion(entries)`**
|
|
60
|
+
- Simulates deletions across multiple model types
|
|
61
|
+
- Aggregates cascade effects into single summary
|
|
62
|
+
- Accepts dict of `{model_class: [instances]}`
|
|
63
|
+
|
|
64
|
+
**Usage Examples:**
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
from winidjango.src.db.bulk import (
|
|
68
|
+
bulk_create_in_steps,
|
|
69
|
+
bulk_create_bulks_in_steps,
|
|
70
|
+
get_differences_between_bulks,
|
|
71
|
+
simulate_bulk_deletion,
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
# Create 10,000 objects in batches of 1000
|
|
75
|
+
authors = [Author(name=f"Author {i}") for i in range(10000)]
|
|
76
|
+
created = bulk_create_in_steps(Author, authors, step=1000)
|
|
77
|
+
# Uses multithreading: ~10x faster than individual saves
|
|
78
|
+
|
|
79
|
+
# Create related models in dependency order
|
|
80
|
+
books = [Book(title=f"Book {i}", author=author) for i, author in enumerate(created)]
|
|
81
|
+
reviews = [Review(book=book, rating=5) for book in books]
|
|
82
|
+
|
|
83
|
+
results = bulk_create_bulks_in_steps({
|
|
84
|
+
Author: authors,
|
|
85
|
+
Book: books, # Created after Author (foreign key dependency)
|
|
86
|
+
Review: reviews, # Created after Book (foreign key dependency)
|
|
87
|
+
})
|
|
88
|
+
# Automatically sorted: Author → Book → Review
|
|
89
|
+
|
|
90
|
+
# Compare two datasets
|
|
91
|
+
from winidjango.src.db.fields import get_fields
|
|
92
|
+
fields = get_fields(Author)
|
|
93
|
+
old_authors = Author.objects.all()
|
|
94
|
+
new_authors = [Author(name=f"Updated {i}") for i in range(100)]
|
|
95
|
+
|
|
96
|
+
to_delete, to_create, unchanged_old, unchanged_new = get_differences_between_bulks(
|
|
97
|
+
list(old_authors), new_authors, fields
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
# Preview deletion impact
|
|
101
|
+
deletion_preview = simulate_bulk_deletion(Author, to_delete)
|
|
102
|
+
# Returns: {Author: {<Author: 1>, <Author: 2>}, Book: {<Book: 1>, <Book: 2>}, ...}
|
|
103
|
+
print(f"Would delete {len(deletion_preview[Author])} authors")
|
|
104
|
+
print(f"Would cascade delete {len(deletion_preview[Book])} books")
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
**Key Features:**
|
|
108
|
+
- **Multithreading**: Parallel processing of chunks for maximum speed
|
|
109
|
+
- **Transaction Safety**: Atomic operations with nested transaction warnings
|
|
110
|
+
- **Configurable Batch Size**: Default 1000, adjustable per operation
|
|
111
|
+
- **Type-Safe**: Full generic type hints with overloads
|
|
112
|
+
- **Memory Efficient**: Processes data in chunks, not all at once
|
|
113
|
+
|
|
114
|
+
#### Model Utilities (`models.py`)
|
|
115
|
+
|
|
116
|
+
**`topological_sort_models(models)`**
|
|
117
|
+
- Sorts models by foreign key dependencies using Python's `graphlib.TopologicalSorter`
|
|
118
|
+
- Ensures correct creation/deletion order
|
|
119
|
+
- Ignores self-referential relationships
|
|
120
|
+
- Raises `CycleError` for circular dependencies
|
|
121
|
+
|
|
122
|
+
**`hash_model_instance(instance, fields)`**
|
|
123
|
+
- Hashes model instances for comparison
|
|
124
|
+
- PK-based for saved instances (fast)
|
|
125
|
+
- Field-based for unsaved instances (content comparison)
|
|
126
|
+
- Used internally by `get_differences_between_bulks()`
|
|
127
|
+
|
|
128
|
+
**`BaseModel`** - Abstract base model with common fields:
|
|
129
|
+
- `created_at` - Auto-populated on creation
|
|
130
|
+
- `updated_at` - Auto-updated on save
|
|
131
|
+
- `meta` property - Type-safe access to `_meta`
|
|
132
|
+
- Custom `__str__()` and `__repr__()`
|
|
133
|
+
|
|
134
|
+
```python
|
|
135
|
+
from winidjango.src.db.models import BaseModel
|
|
136
|
+
|
|
137
|
+
class MyModel(BaseModel):
|
|
138
|
+
name = models.CharField(max_length=100)
|
|
139
|
+
|
|
140
|
+
class Meta(BaseModel.Meta):
|
|
141
|
+
db_table = "my_model"
|
|
142
|
+
|
|
143
|
+
# Automatically includes created_at and updated_at
|
|
144
|
+
obj = MyModel.objects.create(name="test")
|
|
145
|
+
print(obj.created_at) # datetime
|
|
146
|
+
print(obj) # "MyModel(1)"
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
#### Field Utilities (`fields.py`)
|
|
150
|
+
|
|
151
|
+
**`get_fields(model)`** - Get all fields including relationships
|
|
152
|
+
**`get_field_names(fields)`** - Extract field names from field objects
|
|
153
|
+
**`get_model_meta(model)`** - Type-safe access to model `_meta`
|
|
154
|
+
|
|
155
|
+
```python
|
|
156
|
+
from winidjango.src.db.fields import get_fields, get_field_names
|
|
157
|
+
|
|
158
|
+
fields = get_fields(User)
|
|
159
|
+
field_names = get_field_names(fields)
|
|
160
|
+
# ['id', 'username', 'email', 'groups', 'user_permissions', ...]
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
#### SQL Utilities (`sql.py`)
|
|
164
|
+
|
|
165
|
+
**`execute_sql(sql, params=None)`**
|
|
166
|
+
- Execute raw SQL with safe parameter binding
|
|
167
|
+
- Returns tuple: `(column_names, rows)`
|
|
168
|
+
- Automatic cursor management
|
|
169
|
+
- Protection against SQL injection
|
|
170
|
+
|
|
171
|
+
```python
|
|
172
|
+
from winidjango.src.db.sql import execute_sql
|
|
173
|
+
|
|
174
|
+
columns, rows = execute_sql(
|
|
175
|
+
"SELECT id, username FROM auth_user WHERE is_active = %(active)s",
|
|
176
|
+
params={"active": True}
|
|
177
|
+
)
|
|
178
|
+
# columns: ['id', 'username']
|
|
179
|
+
# rows: [(1, 'admin'), (2, 'user'), ...]
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
### Management Commands (`winidjango.src.commands`)
|
|
183
|
+
|
|
184
|
+
A powerful framework for building Django management commands with built-in best practices, automatic logging, and standardized argument handling.
|
|
185
|
+
|
|
186
|
+
#### `ABCBaseCommand` - Base Command Framework
|
|
187
|
+
|
|
188
|
+
Abstract base class that provides a robust foundation for all Django management commands:
|
|
189
|
+
|
|
190
|
+
**Key Features:**
|
|
191
|
+
- **Template Method Pattern**: Enforces consistent command structure while allowing customization
|
|
192
|
+
- **Automatic Logging**: All method calls are logged with performance tracking via `ABCLoggingMixin`
|
|
193
|
+
- **Built-in Common Arguments**: Pre-configured standard options available to all commands:
|
|
194
|
+
- `--dry_run` - Preview changes without executing
|
|
195
|
+
- `--force` - Force execution of actions
|
|
196
|
+
- `--delete` - Enable deletion operations
|
|
197
|
+
- `--yes` - Auto-confirm all prompts
|
|
198
|
+
- `--timeout` - Set command timeout
|
|
199
|
+
- `--batch_size` - Configure batch processing size
|
|
200
|
+
- `--threads` - Control thread count for parallel processing
|
|
201
|
+
- `--processes` - Control process count for multiprocessing
|
|
202
|
+
- **Type-Safe**: Full type hints with abstract method enforcement at compile-time
|
|
203
|
+
- **Structured Execution Flow**: Separates common setup (`base_handle`) from command-specific logic (`handle_command`)
|
|
204
|
+
|
|
205
|
+
**Usage Pattern:**
|
|
206
|
+
```python
|
|
207
|
+
from winidjango.src.commands.base.base import ABCBaseCommand
|
|
208
|
+
from argparse import ArgumentParser
|
|
209
|
+
|
|
210
|
+
class MyCommand(ABCBaseCommand):
|
|
211
|
+
def add_command_arguments(self, parser: ArgumentParser) -> None:
|
|
212
|
+
"""Add command-specific arguments."""
|
|
213
|
+
parser.add_argument('--input-file', type=str, required=True)
|
|
214
|
+
parser.add_argument('--output-format', choices=['json', 'csv'])
|
|
215
|
+
|
|
216
|
+
def handle_command(self) -> None:
|
|
217
|
+
"""Execute command logic."""
|
|
218
|
+
input_file = self.get_option('input_file')
|
|
219
|
+
dry_run = self.get_option('dry_run') # Built-in argument
|
|
220
|
+
batch_size = self.get_option('batch_size') # Built-in argument
|
|
221
|
+
|
|
222
|
+
if dry_run:
|
|
223
|
+
self.stdout.write('Dry run mode - no changes will be made')
|
|
224
|
+
|
|
225
|
+
# Your command logic here
|
|
226
|
+
self.process_data(input_file, batch_size)
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
#### `ImportDataBaseCommand` - Data Import Framework
|
|
230
|
+
|
|
231
|
+
Specialized command for structured data import workflows with automatic cleaning and bulk creation:
|
|
232
|
+
|
|
233
|
+
**Workflow Steps:**
|
|
234
|
+
1. **Import** (`handle_import()`) - Fetch raw data from any source, returns Polars DataFrame
|
|
235
|
+
2. **Clean** (`get_cleaning_df_cls()`) - Define data cleaning logic using `winiutils.CleaningDF`
|
|
236
|
+
3. **Transform** (`get_bulks_by_model()`) - Convert cleaned DataFrame to Django model instances
|
|
237
|
+
4. **Load** (`import_to_db()`) - Bulk create with automatic dependency resolution via topological sorting
|
|
238
|
+
|
|
239
|
+
**Key Features:**
|
|
240
|
+
- **Polars Integration**: High-performance data processing with Polars DataFrames
|
|
241
|
+
- **Automatic Cleaning**: Leverages `winiutils.CleaningDF` for standardized data cleaning pipeline
|
|
242
|
+
- **Dependency-Aware**: Uses `bulk_create_bulks_in_steps()` to handle foreign key relationships automatically
|
|
243
|
+
- **Inherits All Base Features**: Gets all `ABCBaseCommand` functionality (logging, common args, etc.)
|
|
244
|
+
|
|
245
|
+
**Usage Pattern:**
|
|
246
|
+
```python
|
|
247
|
+
from winidjango.src.commands.import_data import ImportDataBaseCommand
|
|
248
|
+
from winiutils.src.data.dataframe.cleaning import CleaningDF
|
|
249
|
+
import polars as pl
|
|
250
|
+
|
|
251
|
+
class MyCleaningDF(CleaningDF):
|
|
252
|
+
"""Define your data cleaning rules."""
|
|
253
|
+
NAME_COL = "name"
|
|
254
|
+
EMAIL_COL = "email"
|
|
255
|
+
|
|
256
|
+
@classmethod
|
|
257
|
+
def get_rename_map(cls) -> dict[str, str]:
|
|
258
|
+
return {"name": "user_name", "email": "user_email"}
|
|
259
|
+
|
|
260
|
+
@classmethod
|
|
261
|
+
def get_col_dtype_map(cls) -> dict[str, type[pl.DataType]]:
|
|
262
|
+
return {cls.NAME_COL: pl.Utf8, cls.EMAIL_COL: pl.Utf8}
|
|
263
|
+
|
|
264
|
+
# ... other cleaning methods
|
|
265
|
+
|
|
266
|
+
class ImportUsersCommand(ImportDataBaseCommand):
|
|
267
|
+
def handle_import(self) -> pl.DataFrame:
|
|
268
|
+
"""Fetch data from source."""
|
|
269
|
+
return pl.read_csv("users.csv")
|
|
270
|
+
|
|
271
|
+
def get_cleaning_df_cls(self) -> type[CleaningDF]:
|
|
272
|
+
"""Return your cleaning class."""
|
|
273
|
+
return MyCleaningDF
|
|
274
|
+
|
|
275
|
+
def get_bulks_by_model(self, df: pl.DataFrame) -> dict[type[Model], Iterable[Model]]:
|
|
276
|
+
"""Convert cleaned data to model instances."""
|
|
277
|
+
users = [User(name=row["name"], email=row["email"])
|
|
278
|
+
for row in df.iter_rows(named=True)]
|
|
279
|
+
profiles = [Profile(user=user) for user in users]
|
|
280
|
+
|
|
281
|
+
# Automatically created in correct order (User before Profile)
|
|
282
|
+
return {User: users, Profile: profiles}
|
|
283
|
+
```
|
|
284
|
+
|
|
285
|
+
**Benefits:**
|
|
286
|
+
- **Standardized Import Process**: Consistent pattern across all data import commands
|
|
287
|
+
- **Separation of Concerns**: Import, cleaning, and transformation logic clearly separated
|
|
288
|
+
- **Automatic Optimization**: Bulk operations with multithreading and dependency resolution
|
|
289
|
+
- **Data Quality**: Built-in cleaning pipeline ensures data consistency
|
|
290
|
+
- **Testable**: Each step can be tested independently
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "winidjango"
|
|
3
|
+
version = "2.0.12"
|
|
4
|
+
description = "A utils package for django"
|
|
5
|
+
authors = [
|
|
6
|
+
{name = "Winipedia",email = "win.steveker@gmx.de"},
|
|
7
|
+
]
|
|
8
|
+
license = "MIT"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.12"
|
|
11
|
+
dependencies = [
|
|
12
|
+
"django",
|
|
13
|
+
"django-stubs-ext",
|
|
14
|
+
"winiutils",
|
|
15
|
+
]
|
|
16
|
+
|
|
17
|
+
[project.scripts]
|
|
18
|
+
winidjango = "pyrig.dev.cli.cli:main"
|
|
19
|
+
|
|
20
|
+
[dependency-groups]
|
|
21
|
+
dev = [
|
|
22
|
+
"bandit",
|
|
23
|
+
"django-stubs",
|
|
24
|
+
"mypy",
|
|
25
|
+
"pre-commit",
|
|
26
|
+
"pyinstaller",
|
|
27
|
+
"pytest",
|
|
28
|
+
"pytest-django",
|
|
29
|
+
"pytest-mock",
|
|
30
|
+
"ruff",
|
|
31
|
+
"types-defusedxml",
|
|
32
|
+
"types-pyinstaller",
|
|
33
|
+
"types-pyyaml",
|
|
34
|
+
"types-setuptools",
|
|
35
|
+
"types-tqdm",
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
[build-system]
|
|
39
|
+
requires = [
|
|
40
|
+
"uv_build",
|
|
41
|
+
]
|
|
42
|
+
build-backend = "uv_build"
|
|
43
|
+
|
|
44
|
+
[tool.uv.build-backend]
|
|
45
|
+
module-name = "winidjango"
|
|
46
|
+
module-root = ""
|
|
47
|
+
|
|
48
|
+
[tool.ruff]
|
|
49
|
+
exclude = [
|
|
50
|
+
".*",
|
|
51
|
+
"**/migrations/*.py",
|
|
52
|
+
]
|
|
53
|
+
|
|
54
|
+
[tool.ruff.lint]
|
|
55
|
+
select = [
|
|
56
|
+
"ALL",
|
|
57
|
+
]
|
|
58
|
+
ignore = [
|
|
59
|
+
"D203",
|
|
60
|
+
"D213",
|
|
61
|
+
"COM812",
|
|
62
|
+
"ANN401",
|
|
63
|
+
]
|
|
64
|
+
fixable = [
|
|
65
|
+
"ALL",
|
|
66
|
+
]
|
|
67
|
+
|
|
68
|
+
[tool.ruff.lint.per-file-ignores]
|
|
69
|
+
"tests/**/*.py" = [
|
|
70
|
+
"S101",
|
|
71
|
+
]
|
|
72
|
+
|
|
73
|
+
[tool.ruff.lint.pydocstyle]
|
|
74
|
+
convention = "google"
|
|
75
|
+
|
|
76
|
+
[tool.mypy]
|
|
77
|
+
strict = true
|
|
78
|
+
warn_unreachable = true
|
|
79
|
+
show_error_codes = true
|
|
80
|
+
files = "."
|
|
81
|
+
|
|
82
|
+
[tool.pytest.ini_options]
|
|
83
|
+
testpaths = [
|
|
84
|
+
"tests",
|
|
85
|
+
]
|
|
86
|
+
|
|
87
|
+
[tool.bandit]
|
|
88
|
+
exclude_dirs = [
|
|
89
|
+
".*",
|
|
90
|
+
]
|
|
91
|
+
|
|
92
|
+
[tool.bandit.assert_used]
|
|
93
|
+
skips = [
|
|
94
|
+
"*test_*.py",
|
|
95
|
+
]
|