winidjango 2.0.11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of winidjango might be problematic. Click here for more details.
- winidjango/__init__.py +34 -0
- winidjango/dev/__init__.py +1 -0
- winidjango/dev/artifacts/__init__.py +1 -0
- winidjango/dev/artifacts/builders/__init__.py +1 -0
- winidjango/dev/artifacts/resources/__init__.py +1 -0
- winidjango/dev/cli/__init__.py +1 -0
- winidjango/dev/cli/subcommands.py +6 -0
- winidjango/dev/configs/__init__.py +1 -0
- winidjango/dev/configs/configs.py +18 -0
- winidjango/dev/tests/__init__.py +1 -0
- winidjango/dev/tests/fixtures/__init__.py +1 -0
- winidjango/dev/tests/fixtures/fixture.py +7 -0
- winidjango/dev/tests/fixtures/scopes/__init__.py +1 -0
- winidjango/dev/tests/fixtures/scopes/class_.py +8 -0
- winidjango/dev/tests/fixtures/scopes/function.py +8 -0
- winidjango/dev/tests/fixtures/scopes/module.py +8 -0
- winidjango/dev/tests/fixtures/scopes/package.py +8 -0
- winidjango/dev/tests/fixtures/scopes/session.py +8 -0
- winidjango/main.py +19 -0
- winidjango/py.typed +0 -0
- winidjango/src/__init__.py +1 -0
- winidjango/src/commands/__init__.py +1 -0
- winidjango/src/commands/base/__init__.py +1 -0
- winidjango/src/commands/base/base.py +305 -0
- winidjango/src/commands/import_data.py +76 -0
- winidjango/src/db/__init__.py +1 -0
- winidjango/src/db/bulk.py +644 -0
- winidjango/src/db/fields.py +101 -0
- winidjango/src/db/models.py +145 -0
- winidjango/src/db/sql.py +63 -0
- winidjango-2.0.11.dist-info/METADATA +303 -0
- winidjango-2.0.11.dist-info/RECORD +34 -0
- winidjango-2.0.11.dist-info/WHEEL +4 -0
- winidjango-2.0.11.dist-info/entry_points.txt +3 -0
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""Fields module.
|
|
2
|
+
|
|
3
|
+
Utils for working with Django model fields.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from typing import TYPE_CHECKING, Any
|
|
7
|
+
|
|
8
|
+
from django.db.models import Field, Model
|
|
9
|
+
|
|
10
|
+
if TYPE_CHECKING:
|
|
11
|
+
from django.contrib.contenttypes.fields import GenericForeignKey
|
|
12
|
+
from django.db.models.fields.related import ForeignObjectRel
|
|
13
|
+
from django.db.models.options import Options
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def get_field_names(
|
|
17
|
+
fields: "list[Field[Any, Any] | ForeignObjectRel | GenericForeignKey]",
|
|
18
|
+
) -> list[str]:
|
|
19
|
+
"""Get the names of all fields from a Django model including relationships.
|
|
20
|
+
|
|
21
|
+
Retrieves the names of all field objects from a Django model, including
|
|
22
|
+
regular fields, foreign key relationships, reverse foreign key relationships,
|
|
23
|
+
and generic foreign keys. This provides a comprehensive view of all model
|
|
24
|
+
attributes that can be used for introspection, validation, or bulk operations.
|
|
25
|
+
|
|
26
|
+
Args:
|
|
27
|
+
fields (list[Field | ForeignObjectRel | GenericForeignKey]):
|
|
28
|
+
The list of field objects to get names from.
|
|
29
|
+
|
|
30
|
+
Returns:
|
|
31
|
+
list[str]: A list containing the names of all fields.
|
|
32
|
+
|
|
33
|
+
Example:
|
|
34
|
+
>>> from django.contrib.auth.models import User
|
|
35
|
+
>>> fields = get_fields(User)
|
|
36
|
+
>>> field_names = get_field_names(fields)
|
|
37
|
+
>>> 'username' in field_names
|
|
38
|
+
True
|
|
39
|
+
>>> 'email' in field_names
|
|
40
|
+
True
|
|
41
|
+
"""
|
|
42
|
+
return [field.name for field in fields]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def get_model_meta(model: type[Model]) -> "Options[Model]":
|
|
46
|
+
"""Get the Django model metadata options object.
|
|
47
|
+
|
|
48
|
+
Retrieves the _meta attribute from a Django model class, which contains
|
|
49
|
+
metadata about the model including field definitions, table name, and
|
|
50
|
+
other model configuration options. This is a convenience wrapper around
|
|
51
|
+
accessing the private _meta attribute directly.
|
|
52
|
+
|
|
53
|
+
Args:
|
|
54
|
+
model (type[Model]): The Django model class to get metadata from.
|
|
55
|
+
|
|
56
|
+
Returns:
|
|
57
|
+
Options[Model]: The model's metadata options object containing
|
|
58
|
+
field definitions, table information, and other model configuration.
|
|
59
|
+
|
|
60
|
+
Example:
|
|
61
|
+
>>> from django.contrib.auth.models import User
|
|
62
|
+
>>> meta = get_model_meta(User)
|
|
63
|
+
>>> meta.db_table
|
|
64
|
+
'auth_user'
|
|
65
|
+
>>> len(meta.get_fields())
|
|
66
|
+
11
|
|
67
|
+
"""
|
|
68
|
+
return model._meta # noqa: SLF001
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def get_fields[TModel: Model](
|
|
72
|
+
model: type[TModel],
|
|
73
|
+
) -> "list[Field[Any, Any] | ForeignObjectRel | GenericForeignKey]":
|
|
74
|
+
"""Get all fields from a Django model including relationships.
|
|
75
|
+
|
|
76
|
+
Retrieves all field objects from a Django model, including regular fields,
|
|
77
|
+
foreign key relationships, reverse foreign key relationships, and generic
|
|
78
|
+
foreign keys. This provides a comprehensive view of all model attributes
|
|
79
|
+
that can be used for introspection, validation, or bulk operations.
|
|
80
|
+
|
|
81
|
+
Args:
|
|
82
|
+
model (type[Model]): The Django model class to get fields from.
|
|
83
|
+
|
|
84
|
+
Returns:
|
|
85
|
+
list[Field | ForeignObjectRel | GenericForeignKey]: A list
|
|
86
|
+
containing all field objects associated with the model, including:
|
|
87
|
+
- Regular model fields (CharField, IntegerField, etc.)
|
|
88
|
+
- Foreign key fields (ForeignKey, OneToOneField, etc.)
|
|
89
|
+
- Reverse relationship fields (ForeignObjectRel)
|
|
90
|
+
- Generic foreign key fields (GenericForeignKey)
|
|
91
|
+
|
|
92
|
+
Example:
|
|
93
|
+
>>> from django.contrib.auth.models import User
|
|
94
|
+
>>> fields = get_fields(User)
|
|
95
|
+
>>> field_names = [f.name for f in fields if hasattr(f, 'name')]
|
|
96
|
+
>>> 'username' in field_names
|
|
97
|
+
True
|
|
98
|
+
>>> 'email' in field_names
|
|
99
|
+
True
|
|
100
|
+
"""
|
|
101
|
+
return get_model_meta(model).get_fields()
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
"""Database utilities for Django.
|
|
2
|
+
|
|
3
|
+
This module provides utility functions for working with Django models,
|
|
4
|
+
including hashing, topological sorting, and database operations.
|
|
5
|
+
These utilities help with efficient and safe database interactions.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from datetime import datetime
|
|
9
|
+
from graphlib import TopologicalSorter
|
|
10
|
+
from typing import TYPE_CHECKING, Any, Self, cast
|
|
11
|
+
|
|
12
|
+
from django.db.models import DateTimeField, Field, Model
|
|
13
|
+
from django.db.models.fields.related import ForeignKey, ForeignObjectRel
|
|
14
|
+
from django.forms.models import model_to_dict
|
|
15
|
+
|
|
16
|
+
from winidjango.src.db.fields import get_field_names, get_fields
|
|
17
|
+
|
|
18
|
+
if TYPE_CHECKING:
|
|
19
|
+
from django.contrib.contenttypes.fields import GenericForeignKey
|
|
20
|
+
from django.db.models.options import Options
|
|
21
|
+
|
|
22
|
+
import logging
|
|
23
|
+
|
|
24
|
+
logger = logging.getLogger(__name__)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def topological_sort_models[TModel: Model](
|
|
28
|
+
models: list[type[TModel]],
|
|
29
|
+
) -> list[type[TModel]]:
|
|
30
|
+
"""Sort Django models in dependency order using topological sorting.
|
|
31
|
+
|
|
32
|
+
Analyzes foreign key relationships between Django models and returns them
|
|
33
|
+
in an order where dependencies come before dependents. This ensures that
|
|
34
|
+
when performing operations like bulk creation or deletion, models are
|
|
35
|
+
processed in the correct order to avoid foreign key constraint violations.
|
|
36
|
+
|
|
37
|
+
The function uses Python's graphlib.TopologicalSorter to perform the sorting
|
|
38
|
+
based on ForeignKey relationships between the provided models. Only
|
|
39
|
+
relationships between models in the input list are considered.
|
|
40
|
+
|
|
41
|
+
Args:
|
|
42
|
+
models (list[type[Model]]): A list of Django model classes to sort
|
|
43
|
+
based on their foreign key dependencies.
|
|
44
|
+
|
|
45
|
+
Returns:
|
|
46
|
+
list[type[Model]]: The input models sorted in dependency order, where
|
|
47
|
+
models that are referenced by foreign keys appear before models
|
|
48
|
+
that reference them. Self-referential relationships are ignored.
|
|
49
|
+
|
|
50
|
+
Raises:
|
|
51
|
+
graphlib.CycleError: If there are circular dependencies between models
|
|
52
|
+
that cannot be resolved.
|
|
53
|
+
|
|
54
|
+
Example:
|
|
55
|
+
>>> # Assuming Author model has no dependencies
|
|
56
|
+
>>> # and Book model has ForeignKey to Author
|
|
57
|
+
>>> models = [Book, Author]
|
|
58
|
+
>>> sorted_models = topological_sort_models(models)
|
|
59
|
+
>>> sorted_models
|
|
60
|
+
[<class 'Author'>, <class 'Book'>]
|
|
61
|
+
|
|
62
|
+
Note:
|
|
63
|
+
- Only considers ForeignKey relationships, not other field types
|
|
64
|
+
- Self-referential foreign keys are ignored to avoid self-loops
|
|
65
|
+
- Only relationships between models in the input list are considered
|
|
66
|
+
"""
|
|
67
|
+
ts: TopologicalSorter[type[TModel]] = TopologicalSorter()
|
|
68
|
+
|
|
69
|
+
for model in models:
|
|
70
|
+
deps = {
|
|
71
|
+
cast("type[TModel]", field.related_model)
|
|
72
|
+
for field in get_fields(model)
|
|
73
|
+
if isinstance(field, ForeignKey)
|
|
74
|
+
and isinstance(field.related_model, type)
|
|
75
|
+
and field.related_model in models
|
|
76
|
+
and field.related_model is not model
|
|
77
|
+
}
|
|
78
|
+
ts.add(model, *deps)
|
|
79
|
+
|
|
80
|
+
return list(ts.static_order())
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def hash_model_instance(
|
|
84
|
+
instance: Model,
|
|
85
|
+
fields: "list[Field[Any, Any] | ForeignObjectRel | GenericForeignKey]",
|
|
86
|
+
) -> int:
|
|
87
|
+
"""Hash a model instance based on its field values.
|
|
88
|
+
|
|
89
|
+
Generates a hash for a Django model instance by considering the values
|
|
90
|
+
of its fields. This can be useful for comparing instances, especially
|
|
91
|
+
when dealing with related objects or complex data structures. The hash
|
|
92
|
+
is generated by recursively hashing related objects up to a specified
|
|
93
|
+
depth.
|
|
94
|
+
This is not very reliable, use with caution.
|
|
95
|
+
Only use if working with unsafed objects or bulks, as with safed
|
|
96
|
+
|
|
97
|
+
Args:
|
|
98
|
+
instance (Model): The Django model instance to hash
|
|
99
|
+
fields (list[str]): The fields to hash
|
|
100
|
+
|
|
101
|
+
Returns:
|
|
102
|
+
int: The hash value representing the instance's data
|
|
103
|
+
|
|
104
|
+
"""
|
|
105
|
+
if instance.pk:
|
|
106
|
+
return hash(instance.pk)
|
|
107
|
+
|
|
108
|
+
field_names = get_field_names(fields)
|
|
109
|
+
model_dict = model_to_dict(instance, fields=field_names)
|
|
110
|
+
sorted_dict = dict(sorted(model_dict.items()))
|
|
111
|
+
values = (type(instance), tuple(sorted_dict.items()))
|
|
112
|
+
return hash(values)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
class BaseModel(Model):
|
|
116
|
+
"""Base model for all models in the project.
|
|
117
|
+
|
|
118
|
+
Provides common fields and methods for all models.
|
|
119
|
+
"""
|
|
120
|
+
|
|
121
|
+
created_at: DateTimeField[datetime, datetime] = DateTimeField(auto_now_add=True)
|
|
122
|
+
updated_at: DateTimeField[datetime, datetime] = DateTimeField(auto_now=True)
|
|
123
|
+
|
|
124
|
+
class Meta:
|
|
125
|
+
"""Mark the model as abstract."""
|
|
126
|
+
|
|
127
|
+
# abstract does not inherit in children
|
|
128
|
+
abstract = True
|
|
129
|
+
|
|
130
|
+
def __str__(self) -> str:
|
|
131
|
+
"""Base string representation of a model.
|
|
132
|
+
|
|
133
|
+
Returns:
|
|
134
|
+
str: The string representation of the model as all fields and their values.
|
|
135
|
+
"""
|
|
136
|
+
return f"{self.__class__.__name__}({self.pk})"
|
|
137
|
+
|
|
138
|
+
def __repr__(self) -> str:
|
|
139
|
+
"""Base representation of a model."""
|
|
140
|
+
return str(self)
|
|
141
|
+
|
|
142
|
+
@property
|
|
143
|
+
def meta(self) -> "Options[Self]":
|
|
144
|
+
"""Get the meta options for the model."""
|
|
145
|
+
return self._meta
|
winidjango/src/db/sql.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""Module for database operations with sql."""
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from django.db import connection
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def execute_sql(
|
|
9
|
+
sql: str, params: dict[str, Any] | None = None
|
|
10
|
+
) -> tuple[list[str], list[Any]]:
|
|
11
|
+
"""Execute raw SQL query and return column names with results.
|
|
12
|
+
|
|
13
|
+
Executes a raw SQL query using Django's database connection and returns
|
|
14
|
+
both the column names and the result rows. This provides a convenient
|
|
15
|
+
way to run custom SQL queries while maintaining Django's database
|
|
16
|
+
connection management and parameter binding for security.
|
|
17
|
+
|
|
18
|
+
The function automatically handles cursor management and ensures proper
|
|
19
|
+
cleanup of database resources. Parameters are safely bound to prevent
|
|
20
|
+
SQL injection attacks.
|
|
21
|
+
|
|
22
|
+
Args:
|
|
23
|
+
sql (str): The SQL query string to execute. Can contain parameter
|
|
24
|
+
placeholders that will be safely bound using the params argument.
|
|
25
|
+
params (dict[str, Any] | None, optional): Dictionary of parameters
|
|
26
|
+
to bind to the SQL query for safe parameter substitution.
|
|
27
|
+
Defaults to None if no parameters are needed.
|
|
28
|
+
|
|
29
|
+
Returns:
|
|
30
|
+
tuple[list[str], list[Any]]: A tuple containing:
|
|
31
|
+
- list[str]: Column names from the query result
|
|
32
|
+
- list[Any]: List of result rows, where each row is a tuple
|
|
33
|
+
of values corresponding to the column names
|
|
34
|
+
Empty list if no results are returned
|
|
35
|
+
|
|
36
|
+
Raises:
|
|
37
|
+
django.db.Error: If there's a database error during query execution
|
|
38
|
+
django.db.ProgrammingError: If the SQL syntax is invalid
|
|
39
|
+
django.db.IntegrityError: If the query violates database constraints
|
|
40
|
+
|
|
41
|
+
Example:
|
|
42
|
+
>>> sql = "SELECT id, username FROM auth_user WHERE is_active = %(active)s"
|
|
43
|
+
>>> params = {"active": True}
|
|
44
|
+
>>> columns, rows = execute_sql(sql, params)
|
|
45
|
+
>>> columns
|
|
46
|
+
['id', 'username']
|
|
47
|
+
>>> rows[0]
|
|
48
|
+
(1, 'admin')
|
|
49
|
+
|
|
50
|
+
Note:
|
|
51
|
+
- Uses Django's default database connection
|
|
52
|
+
- Automatically manages cursor lifecycle
|
|
53
|
+
- Parameters are safely bound to prevent SQL injection
|
|
54
|
+
- Returns all results in memory - use with caution for large datasets
|
|
55
|
+
"""
|
|
56
|
+
with connection.cursor() as cursor:
|
|
57
|
+
cursor.execute(sql=sql, params=params)
|
|
58
|
+
rows = cursor.fetchall()
|
|
59
|
+
column_names = (
|
|
60
|
+
[col[0] for col in cursor.description] if cursor.description else []
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
return column_names, rows
|
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: winidjango
|
|
3
|
+
Version: 2.0.11
|
|
4
|
+
Summary: A utils package for django
|
|
5
|
+
Author: Winipedia
|
|
6
|
+
Author-email: Winipedia <win.steveker@gmx.de>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
Requires-Dist: django
|
|
9
|
+
Requires-Dist: django-stubs-ext
|
|
10
|
+
Requires-Dist: winiutils
|
|
11
|
+
Requires-Python: >=3.12
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
|
|
14
|
+
# winidjango
|
|
15
|
+
|
|
16
|
+
(This project uses [pyrig](https://github.com/Winipedia/pyrig))
|
|
17
|
+
|
|
18
|
+
## Overview
|
|
19
|
+
|
|
20
|
+
**winidjango** is a production-ready Django utilities library that simplifies complex database operations and provides structured patterns for data management tasks. Built with type safety and performance in mind, it leverages modern Python features and integrates seamlessly with Django's ORM.
|
|
21
|
+
|
|
22
|
+
## Features
|
|
23
|
+
|
|
24
|
+
### Database Utilities (`winidjango.src.db`)
|
|
25
|
+
|
|
26
|
+
High-performance database operations with automatic optimization, dependency management, and type safety.
|
|
27
|
+
|
|
28
|
+
#### Bulk Operations (`bulk.py`)
|
|
29
|
+
|
|
30
|
+
Efficiently process large datasets with automatic chunking, multithreading, and transaction management.
|
|
31
|
+
|
|
32
|
+
**Core Functions:**
|
|
33
|
+
|
|
34
|
+
**`bulk_create_in_steps(model, bulk, step=1000)`**
|
|
35
|
+
- Creates thousands of model instances in configurable batches (default: 1000)
|
|
36
|
+
- Uses multithreading for parallel processing across chunks
|
|
37
|
+
- Returns list of created instances with populated PKs
|
|
38
|
+
- Wrapped in atomic transactions for data integrity
|
|
39
|
+
|
|
40
|
+
**`bulk_update_in_steps(model, bulk, update_fields, step=1000)`**
|
|
41
|
+
- Updates large datasets efficiently in batches
|
|
42
|
+
- Requires explicit `update_fields` list for safety
|
|
43
|
+
- Returns total count of updated objects
|
|
44
|
+
- Multithreaded processing for maximum performance
|
|
45
|
+
|
|
46
|
+
**`bulk_delete_in_steps(model, bulk, step=1000)`**
|
|
47
|
+
- Deletes in batches with cascade tracking
|
|
48
|
+
- Returns tuple: `(total_count, {model_name: count})`
|
|
49
|
+
- Tracks all cascade deletions across related models
|
|
50
|
+
- Safe handling of foreign key constraints
|
|
51
|
+
|
|
52
|
+
**`bulk_create_bulks_in_steps(bulk_by_class, step=1000)`**
|
|
53
|
+
- **Automatic Dependency Resolution**: Creates multiple model types in correct order
|
|
54
|
+
- Uses topological sorting to handle foreign key relationships
|
|
55
|
+
- Accepts dict mapping model classes to instance lists
|
|
56
|
+
- Returns dict with created instances (PKs populated)
|
|
57
|
+
|
|
58
|
+
**Advanced Comparison & Simulation:**
|
|
59
|
+
|
|
60
|
+
**`get_differences_between_bulks(bulk1, bulk2, fields)`**
|
|
61
|
+
- Compares two bulks by hashing field values
|
|
62
|
+
- Returns 4-tuple: `(in_1_not_2, in_2_not_1, in_both_from_1, in_both_from_2)`
|
|
63
|
+
- Useful for sync operations and change detection
|
|
64
|
+
- Preserves original object references
|
|
65
|
+
|
|
66
|
+
**`simulate_bulk_deletion(model_class, entries)`**
|
|
67
|
+
- **Preview deletions without executing** using Django's Collector
|
|
68
|
+
- Returns dict mapping models to objects that would be deleted
|
|
69
|
+
- Includes all cascade deletions
|
|
70
|
+
- Perfect for "what-if" analysis before destructive operations
|
|
71
|
+
|
|
72
|
+
**`multi_simulate_bulk_deletion(entries)`**
|
|
73
|
+
- Simulates deletions across multiple model types
|
|
74
|
+
- Aggregates cascade effects into single summary
|
|
75
|
+
- Accepts dict of `{model_class: [instances]}`
|
|
76
|
+
|
|
77
|
+
**Usage Examples:**
|
|
78
|
+
|
|
79
|
+
```python
|
|
80
|
+
from winidjango.src.db.bulk import (
|
|
81
|
+
bulk_create_in_steps,
|
|
82
|
+
bulk_create_bulks_in_steps,
|
|
83
|
+
get_differences_between_bulks,
|
|
84
|
+
simulate_bulk_deletion,
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
# Create 10,000 objects in batches of 1000
|
|
88
|
+
authors = [Author(name=f"Author {i}") for i in range(10000)]
|
|
89
|
+
created = bulk_create_in_steps(Author, authors, step=1000)
|
|
90
|
+
# Uses multithreading: ~10x faster than individual saves
|
|
91
|
+
|
|
92
|
+
# Create related models in dependency order
|
|
93
|
+
books = [Book(title=f"Book {i}", author=author) for i, author in enumerate(created)]
|
|
94
|
+
reviews = [Review(book=book, rating=5) for book in books]
|
|
95
|
+
|
|
96
|
+
results = bulk_create_bulks_in_steps({
|
|
97
|
+
Author: authors,
|
|
98
|
+
Book: books, # Created after Author (foreign key dependency)
|
|
99
|
+
Review: reviews, # Created after Book (foreign key dependency)
|
|
100
|
+
})
|
|
101
|
+
# Automatically sorted: Author → Book → Review
|
|
102
|
+
|
|
103
|
+
# Compare two datasets
|
|
104
|
+
from winidjango.src.db.fields import get_fields
|
|
105
|
+
fields = get_fields(Author)
|
|
106
|
+
old_authors = Author.objects.all()
|
|
107
|
+
new_authors = [Author(name=f"Updated {i}") for i in range(100)]
|
|
108
|
+
|
|
109
|
+
to_delete, to_create, unchanged_old, unchanged_new = get_differences_between_bulks(
|
|
110
|
+
list(old_authors), new_authors, fields
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
# Preview deletion impact
|
|
114
|
+
deletion_preview = simulate_bulk_deletion(Author, to_delete)
|
|
115
|
+
# Returns: {Author: {<Author: 1>, <Author: 2>}, Book: {<Book: 1>, <Book: 2>}, ...}
|
|
116
|
+
print(f"Would delete {len(deletion_preview[Author])} authors")
|
|
117
|
+
print(f"Would cascade delete {len(deletion_preview[Book])} books")
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
**Key Features:**
|
|
121
|
+
- **Multithreading**: Parallel processing of chunks for maximum speed
|
|
122
|
+
- **Transaction Safety**: Atomic operations with nested transaction warnings
|
|
123
|
+
- **Configurable Batch Size**: Default 1000, adjustable per operation
|
|
124
|
+
- **Type-Safe**: Full generic type hints with overloads
|
|
125
|
+
- **Memory Efficient**: Processes data in chunks, not all at once
|
|
126
|
+
|
|
127
|
+
#### Model Utilities (`models.py`)
|
|
128
|
+
|
|
129
|
+
**`topological_sort_models(models)`**
|
|
130
|
+
- Sorts models by foreign key dependencies using Python's `graphlib.TopologicalSorter`
|
|
131
|
+
- Ensures correct creation/deletion order
|
|
132
|
+
- Ignores self-referential relationships
|
|
133
|
+
- Raises `CycleError` for circular dependencies
|
|
134
|
+
|
|
135
|
+
**`hash_model_instance(instance, fields)`**
|
|
136
|
+
- Hashes model instances for comparison
|
|
137
|
+
- PK-based for saved instances (fast)
|
|
138
|
+
- Field-based for unsaved instances (content comparison)
|
|
139
|
+
- Used internally by `get_differences_between_bulks()`
|
|
140
|
+
|
|
141
|
+
**`BaseModel`** - Abstract base model with common fields:
|
|
142
|
+
- `created_at` - Auto-populated on creation
|
|
143
|
+
- `updated_at` - Auto-updated on save
|
|
144
|
+
- `meta` property - Type-safe access to `_meta`
|
|
145
|
+
- Custom `__str__()` and `__repr__()`
|
|
146
|
+
|
|
147
|
+
```python
|
|
148
|
+
from winidjango.src.db.models import BaseModel
|
|
149
|
+
|
|
150
|
+
class MyModel(BaseModel):
|
|
151
|
+
name = models.CharField(max_length=100)
|
|
152
|
+
|
|
153
|
+
class Meta(BaseModel.Meta):
|
|
154
|
+
db_table = "my_model"
|
|
155
|
+
|
|
156
|
+
# Automatically includes created_at and updated_at
|
|
157
|
+
obj = MyModel.objects.create(name="test")
|
|
158
|
+
print(obj.created_at) # datetime
|
|
159
|
+
print(obj) # "MyModel(1)"
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
#### Field Utilities (`fields.py`)
|
|
163
|
+
|
|
164
|
+
**`get_fields(model)`** - Get all fields including relationships
|
|
165
|
+
**`get_field_names(fields)`** - Extract field names from field objects
|
|
166
|
+
**`get_model_meta(model)`** - Type-safe access to model `_meta`
|
|
167
|
+
|
|
168
|
+
```python
|
|
169
|
+
from winidjango.src.db.fields import get_fields, get_field_names
|
|
170
|
+
|
|
171
|
+
fields = get_fields(User)
|
|
172
|
+
field_names = get_field_names(fields)
|
|
173
|
+
# ['id', 'username', 'email', 'groups', 'user_permissions', ...]
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
#### SQL Utilities (`sql.py`)
|
|
177
|
+
|
|
178
|
+
**`execute_sql(sql, params=None)`**
|
|
179
|
+
- Execute raw SQL with safe parameter binding
|
|
180
|
+
- Returns tuple: `(column_names, rows)`
|
|
181
|
+
- Automatic cursor management
|
|
182
|
+
- Protection against SQL injection
|
|
183
|
+
|
|
184
|
+
```python
|
|
185
|
+
from winidjango.src.db.sql import execute_sql
|
|
186
|
+
|
|
187
|
+
columns, rows = execute_sql(
|
|
188
|
+
"SELECT id, username FROM auth_user WHERE is_active = %(active)s",
|
|
189
|
+
params={"active": True}
|
|
190
|
+
)
|
|
191
|
+
# columns: ['id', 'username']
|
|
192
|
+
# rows: [(1, 'admin'), (2, 'user'), ...]
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
### Management Commands (`winidjango.src.commands`)
|
|
196
|
+
|
|
197
|
+
A powerful framework for building Django management commands with built-in best practices, automatic logging, and standardized argument handling.
|
|
198
|
+
|
|
199
|
+
#### `ABCBaseCommand` - Base Command Framework
|
|
200
|
+
|
|
201
|
+
Abstract base class that provides a robust foundation for all Django management commands:
|
|
202
|
+
|
|
203
|
+
**Key Features:**
|
|
204
|
+
- **Template Method Pattern**: Enforces consistent command structure while allowing customization
|
|
205
|
+
- **Automatic Logging**: All method calls are logged with performance tracking via `ABCLoggingMixin`
|
|
206
|
+
- **Built-in Common Arguments**: Pre-configured standard options available to all commands:
|
|
207
|
+
- `--dry_run` - Preview changes without executing
|
|
208
|
+
- `--force` - Force execution of actions
|
|
209
|
+
- `--delete` - Enable deletion operations
|
|
210
|
+
- `--yes` - Auto-confirm all prompts
|
|
211
|
+
- `--timeout` - Set command timeout
|
|
212
|
+
- `--batch_size` - Configure batch processing size
|
|
213
|
+
- `--threads` - Control thread count for parallel processing
|
|
214
|
+
- `--processes` - Control process count for multiprocessing
|
|
215
|
+
- **Type-Safe**: Full type hints with abstract method enforcement at compile-time
|
|
216
|
+
- **Structured Execution Flow**: Separates common setup (`base_handle`) from command-specific logic (`handle_command`)
|
|
217
|
+
|
|
218
|
+
**Usage Pattern:**
|
|
219
|
+
```python
|
|
220
|
+
from winidjango.src.commands.base.base import ABCBaseCommand
|
|
221
|
+
from argparse import ArgumentParser
|
|
222
|
+
|
|
223
|
+
class MyCommand(ABCBaseCommand):
|
|
224
|
+
def add_command_arguments(self, parser: ArgumentParser) -> None:
|
|
225
|
+
"""Add command-specific arguments."""
|
|
226
|
+
parser.add_argument('--input-file', type=str, required=True)
|
|
227
|
+
parser.add_argument('--output-format', choices=['json', 'csv'])
|
|
228
|
+
|
|
229
|
+
def handle_command(self) -> None:
|
|
230
|
+
"""Execute command logic."""
|
|
231
|
+
input_file = self.get_option('input_file')
|
|
232
|
+
dry_run = self.get_option('dry_run') # Built-in argument
|
|
233
|
+
batch_size = self.get_option('batch_size') # Built-in argument
|
|
234
|
+
|
|
235
|
+
if dry_run:
|
|
236
|
+
self.stdout.write('Dry run mode - no changes will be made')
|
|
237
|
+
|
|
238
|
+
# Your command logic here
|
|
239
|
+
self.process_data(input_file, batch_size)
|
|
240
|
+
```
|
|
241
|
+
|
|
242
|
+
#### `ImportDataBaseCommand` - Data Import Framework
|
|
243
|
+
|
|
244
|
+
Specialized command for structured data import workflows with automatic cleaning and bulk creation:
|
|
245
|
+
|
|
246
|
+
**Workflow Steps:**
|
|
247
|
+
1. **Import** (`handle_import()`) - Fetch raw data from any source, returns Polars DataFrame
|
|
248
|
+
2. **Clean** (`get_cleaning_df_cls()`) - Define data cleaning logic using `winiutils.CleaningDF`
|
|
249
|
+
3. **Transform** (`get_bulks_by_model()`) - Convert cleaned DataFrame to Django model instances
|
|
250
|
+
4. **Load** (`import_to_db()`) - Bulk create with automatic dependency resolution via topological sorting
|
|
251
|
+
|
|
252
|
+
**Key Features:**
|
|
253
|
+
- **Polars Integration**: High-performance data processing with Polars DataFrames
|
|
254
|
+
- **Automatic Cleaning**: Leverages `winiutils.CleaningDF` for standardized data cleaning pipeline
|
|
255
|
+
- **Dependency-Aware**: Uses `bulk_create_bulks_in_steps()` to handle foreign key relationships automatically
|
|
256
|
+
- **Inherits All Base Features**: Gets all `ABCBaseCommand` functionality (logging, common args, etc.)
|
|
257
|
+
|
|
258
|
+
**Usage Pattern:**
|
|
259
|
+
```python
|
|
260
|
+
from winidjango.src.commands.import_data import ImportDataBaseCommand
|
|
261
|
+
from winiutils.src.data.dataframe.cleaning import CleaningDF
|
|
262
|
+
import polars as pl
|
|
263
|
+
|
|
264
|
+
class MyCleaningDF(CleaningDF):
|
|
265
|
+
"""Define your data cleaning rules."""
|
|
266
|
+
NAME_COL = "name"
|
|
267
|
+
EMAIL_COL = "email"
|
|
268
|
+
|
|
269
|
+
@classmethod
|
|
270
|
+
def get_rename_map(cls) -> dict[str, str]:
|
|
271
|
+
return {"name": "user_name", "email": "user_email"}
|
|
272
|
+
|
|
273
|
+
@classmethod
|
|
274
|
+
def get_col_dtype_map(cls) -> dict[str, type[pl.DataType]]:
|
|
275
|
+
return {cls.NAME_COL: pl.Utf8, cls.EMAIL_COL: pl.Utf8}
|
|
276
|
+
|
|
277
|
+
# ... other cleaning methods
|
|
278
|
+
|
|
279
|
+
class ImportUsersCommand(ImportDataBaseCommand):
|
|
280
|
+
def handle_import(self) -> pl.DataFrame:
|
|
281
|
+
"""Fetch data from source."""
|
|
282
|
+
return pl.read_csv("users.csv")
|
|
283
|
+
|
|
284
|
+
def get_cleaning_df_cls(self) -> type[CleaningDF]:
|
|
285
|
+
"""Return your cleaning class."""
|
|
286
|
+
return MyCleaningDF
|
|
287
|
+
|
|
288
|
+
def get_bulks_by_model(self, df: pl.DataFrame) -> dict[type[Model], Iterable[Model]]:
|
|
289
|
+
"""Convert cleaned data to model instances."""
|
|
290
|
+
users = [User(name=row["name"], email=row["email"])
|
|
291
|
+
for row in df.iter_rows(named=True)]
|
|
292
|
+
profiles = [Profile(user=user) for user in users]
|
|
293
|
+
|
|
294
|
+
# Automatically created in correct order (User before Profile)
|
|
295
|
+
return {User: users, Profile: profiles}
|
|
296
|
+
```
|
|
297
|
+
|
|
298
|
+
**Benefits:**
|
|
299
|
+
- **Standardized Import Process**: Consistent pattern across all data import commands
|
|
300
|
+
- **Separation of Concerns**: Import, cleaning, and transformation logic clearly separated
|
|
301
|
+
- **Automatic Optimization**: Bulk operations with multithreading and dependency resolution
|
|
302
|
+
- **Data Quality**: Built-in cleaning pipeline ensures data consistency
|
|
303
|
+
- **Testable**: Each step can be tested independently
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
winidjango/__init__.py,sha256=pxCy4ywQWxN3MKjm9U2vcIDz-nsvDLgNzsE9IOq15DQ,823
|
|
2
|
+
winidjango/dev/__init__.py,sha256=XHsbmjiaGom-KX-S3leCY9cJD3aP9p_0X6xYMcdkHBU,23
|
|
3
|
+
winidjango/dev/artifacts/__init__.py,sha256=XHsbmjiaGom-KX-S3leCY9cJD3aP9p_0X6xYMcdkHBU,23
|
|
4
|
+
winidjango/dev/artifacts/builders/__init__.py,sha256=XHsbmjiaGom-KX-S3leCY9cJD3aP9p_0X6xYMcdkHBU,23
|
|
5
|
+
winidjango/dev/artifacts/resources/__init__.py,sha256=XHsbmjiaGom-KX-S3leCY9cJD3aP9p_0X6xYMcdkHBU,23
|
|
6
|
+
winidjango/dev/cli/__init__.py,sha256=XHsbmjiaGom-KX-S3leCY9cJD3aP9p_0X6xYMcdkHBU,23
|
|
7
|
+
winidjango/dev/cli/subcommands.py,sha256=iurWZwJwEKAfGpfjkn1YOhnRbIruCB4ouE-8R_Lh3JY,228
|
|
8
|
+
winidjango/dev/configs/__init__.py,sha256=XHsbmjiaGom-KX-S3leCY9cJD3aP9p_0X6xYMcdkHBU,23
|
|
9
|
+
winidjango/dev/configs/configs.py,sha256=N85kVaRHCHXc-ny0jbuYQwTcrxQJx_X_BcG30IcyrGw,586
|
|
10
|
+
winidjango/dev/tests/__init__.py,sha256=XHsbmjiaGom-KX-S3leCY9cJD3aP9p_0X6xYMcdkHBU,23
|
|
11
|
+
winidjango/dev/tests/fixtures/__init__.py,sha256=XHsbmjiaGom-KX-S3leCY9cJD3aP9p_0X6xYMcdkHBU,23
|
|
12
|
+
winidjango/dev/tests/fixtures/fixture.py,sha256=fA8GaETQ0foL1CEqvmxf6ab2Wz-_CxxI-ttkj8CUmok,263
|
|
13
|
+
winidjango/dev/tests/fixtures/scopes/__init__.py,sha256=XHsbmjiaGom-KX-S3leCY9cJD3aP9p_0X6xYMcdkHBU,23
|
|
14
|
+
winidjango/dev/tests/fixtures/scopes/class_.py,sha256=ZuI8_LEGy8SiutoygpgQq5_ojjlJRP-fNxDuhEzLj7M,417
|
|
15
|
+
winidjango/dev/tests/fixtures/scopes/function.py,sha256=2ehbrDhtw581pKmvY0KvuDkPCma8E19TyZboOb80L2I,428
|
|
16
|
+
winidjango/dev/tests/fixtures/scopes/module.py,sha256=OwdimjLzNMnsFT2th3OW5ZKP2YzjelC2NJ6jhSPrMMo,420
|
|
17
|
+
winidjango/dev/tests/fixtures/scopes/package.py,sha256=yTDD_7UcWdml7uahvIQ4XQKVeVl-TqAAFtbe9t9VG9c,424
|
|
18
|
+
winidjango/dev/tests/fixtures/scopes/session.py,sha256=MSN62vUCdpSPWBl9wYrEWu4qAmqwyINscgat2Rocp9M,423
|
|
19
|
+
winidjango/main.py,sha256=-w752ghb91kXeuIO1rvOiN_nGypBpQqLCJxVqMBP82c,409
|
|
20
|
+
winidjango/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
21
|
+
winidjango/src/__init__.py,sha256=KghXjKc_HAjcpkCULdXY9dN3SK2GIiYNPa3V5bCDVkE,19
|
|
22
|
+
winidjango/src/commands/__init__.py,sha256=XHsbmjiaGom-KX-S3leCY9cJD3aP9p_0X6xYMcdkHBU,23
|
|
23
|
+
winidjango/src/commands/base/__init__.py,sha256=XHsbmjiaGom-KX-S3leCY9cJD3aP9p_0X6xYMcdkHBU,23
|
|
24
|
+
winidjango/src/commands/base/base.py,sha256=I4fevCwaMgHeDVt9YSSstvvxs44kGD4V09OtAOPNRJE,12716
|
|
25
|
+
winidjango/src/commands/import_data.py,sha256=-heq2183v-vgMoOTOwLmYZ3-tRg68H9Haia3sYa5270,2484
|
|
26
|
+
winidjango/src/db/__init__.py,sha256=XHsbmjiaGom-KX-S3leCY9cJD3aP9p_0X6xYMcdkHBU,23
|
|
27
|
+
winidjango/src/db/bulk.py,sha256=Rpw8pUK-swUMm2OrzR5XUzs4Gfx_OB0LvKQus13i9js,22468
|
|
28
|
+
winidjango/src/db/fields.py,sha256=o_gJlb4D7FmNh9smL8qv0SbEeISjd4WXTu4fDQHNJC8,3616
|
|
29
|
+
winidjango/src/db/models.py,sha256=dV6ZdayNFfUCHB6Gpsp_V4a1ZiciKzsQXZ83oW_gj-Q,5052
|
|
30
|
+
winidjango/src/db/sql.py,sha256=MG9iTUTJCetCKGf75k-EyacUzbb-G4u_-NL9NTngFrk,2446
|
|
31
|
+
winidjango-2.0.11.dist-info/WHEEL,sha256=z-mOpxbJHqy3cq6SvUThBZdaLGFZzdZPtgWLcP2NKjQ,79
|
|
32
|
+
winidjango-2.0.11.dist-info/entry_points.txt,sha256=qM1ENsRWSLulhF8Cy92TeHW-4v8SAgzkRW7hNHXHG-4,55
|
|
33
|
+
winidjango-2.0.11.dist-info/METADATA,sha256=0PFNLyUWIZkG8-wZ3gp2iA4OXM5wMX3ukxAXGciAYhY,11662
|
|
34
|
+
winidjango-2.0.11.dist-info/RECORD,,
|