djanquiltdb 3.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- djanquiltdb-3.0.0/LICENSE +28 -0
- djanquiltdb-3.0.0/MANIFEST.in +7 -0
- djanquiltdb-3.0.0/PKG-INFO +52 -0
- djanquiltdb-3.0.0/README.rst +60 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/__init__.py +26 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/apps.py +202 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/collector.py +103 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/contrib/__init__.py +0 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/contrib/quilt_admin/__init__.py +0 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/contrib/quilt_admin/apps.py +34 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/contrib/quilt_admin/context_processors.py +85 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/contrib/quilt_admin/middleware.py +196 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/contrib/quilt_admin/shard_selector.py +78 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/contrib/quilt_admin/templates/admin/base_site.html +72 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/contrib/quilt_admin/urls.py +7 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/contrib/quilt_admin/utils.py +27 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/contrib/quilt_admin/views.py +94 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/contrib/quilt_auth/__init__.py +1 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/contrib/quilt_auth/apps.py +6 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/contrib/quilt_auth/management/__init__.py +0 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/contrib/quilt_auth/management/commands/__init__.py +0 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/contrib/quilt_auth/management/commands/createsuperuser.py +119 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/db/__init__.py +31 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/decorators.py +409 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/exceptions.py +2 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/forms.py +18 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/management/__init__.py +0 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/management/base.py +39 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/management/commands/__init__.py +0 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/management/commands/flush.py +94 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/management/commands/loaddata.py +308 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/management/commands/migrate.py +15 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/management/commands/migrate_shards.py +387 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/management/commands/move_data_to_shard.py +522 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/management/commands/move_shard_to_node.py +535 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/management/commands/move_sharded_models.py +182 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/management/commands/purge_schema.py +117 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/management/commands/purge_shard_data.py +196 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/management/commands/sqlflush.py +62 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/middleware.py +168 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/models.py +117 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/options.py +103 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/postgresql_backend/__init__.py +1 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/postgresql_backend/base.py +712 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/postgresql_backend/creation.py +74 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/postgresql_backend/introspection.py +294 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/postgresql_backend/operations.py +45 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/postgresql_backend/utils.py +73 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/router.py +123 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/sessions.py +128 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/transaction.py +48 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/utils.py +717 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb.egg-info/PKG-INFO +52 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb.egg-info/SOURCES.txt +63 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb.egg-info/dependency_links.txt +1 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb.egg-info/requires.txt +27 -0
- djanquiltdb-3.0.0/djanquiltdb/djanquiltdb.egg-info/top_level.txt +1 -0
- djanquiltdb-3.0.0/djanquiltdb/templates/example/connection_exception.html +6 -0
- djanquiltdb-3.0.0/djanquiltdb/templates/example/home.html +11 -0
- djanquiltdb-3.0.0/djanquiltdb/templates/example/login.html +10 -0
- djanquiltdb-3.0.0/djanquiltdb/templates/example/state_exception.html +6 -0
- djanquiltdb-3.0.0/pyproject.toml +15 -0
- djanquiltdb-3.0.0/setup.cfg +7 -0
- djanquiltdb-3.0.0/setup.py +63 -0
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
Copyright (c) 2017-2023, Patchman B.V.
|
|
2
|
+
Copyright (c) 2023-2025, Cloud Linux Software, Inc.
|
|
3
|
+
Copyright (c) 2025-2026, DjanQuiltDB Project
|
|
4
|
+
|
|
5
|
+
All rights reserved.
|
|
6
|
+
|
|
7
|
+
Redistribution and use in source and binary forms, with or without
|
|
8
|
+
modification, are permitted provided that the following conditions are met:
|
|
9
|
+
* Redistributions of source code must retain the above copyright
|
|
10
|
+
notice, this list of conditions and the following disclaimer.
|
|
11
|
+
* Redistributions in binary form must reproduce the above copyright
|
|
12
|
+
notice, this list of conditions and the following disclaimer in the
|
|
13
|
+
documentation and/or other materials provided with the distribution.
|
|
14
|
+
* Neither the names of Patchman B.V., Cloud Linux Software, Inc. or
|
|
15
|
+
DjanQuiltDB, nor the names of their contributors may be used to
|
|
16
|
+
endorse or promote products derived from this software without specific
|
|
17
|
+
prior written permission.
|
|
18
|
+
|
|
19
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
|
20
|
+
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
|
21
|
+
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
22
|
+
DISCLAIMED. IN NO EVENT SHALL PATCHMAN B.V., CLOUD LINUX SOFTWARE, INC., OR
|
|
23
|
+
THE DJANQUILTDB PROJECT BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
|
|
24
|
+
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
|
|
25
|
+
OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
26
|
+
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
|
27
|
+
STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
28
|
+
OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: djanquiltdb
|
|
3
|
+
Version: 3.0.0
|
|
4
|
+
Summary: Library to shard a database on the hierarchy's top level table.
|
|
5
|
+
Home-page: https://www.github.com/DjanQuiltDB/djanquiltdb
|
|
6
|
+
Author: DjanQuiltDB Project; Cloud Linux Software, Inc.; Patchman B.V.
|
|
7
|
+
Author-email: djanquiltdb@portal42.net; info@cloudlinux.com; hello@patchman.co
|
|
8
|
+
License: BSD-3-Clause
|
|
9
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
10
|
+
Classifier: Environment :: Web Environment
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Programming Language :: Python
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
16
|
+
Classifier: Framework :: Django
|
|
17
|
+
Classifier: Framework :: Django :: 6.0
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Requires-Dist: django<7.0,>=6.0
|
|
20
|
+
Requires-Dist: psycopg[binary]>=3.0.0
|
|
21
|
+
Requires-Dist: progressbar2
|
|
22
|
+
Provides-Extra: test
|
|
23
|
+
Requires-Dist: tox==4.12.1; extra == "test"
|
|
24
|
+
Requires-Dist: pluggy; extra == "test"
|
|
25
|
+
Requires-Dist: dj-database-url==2.1.0; extra == "test"
|
|
26
|
+
Requires-Dist: django-braces==1.15.0; extra == "test"
|
|
27
|
+
Requires-Dist: tblib; extra == "test"
|
|
28
|
+
Requires-Dist: filelock; extra == "test"
|
|
29
|
+
Requires-Dist: coverage; extra == "test"
|
|
30
|
+
Requires-Dist: PyYAML; extra == "test"
|
|
31
|
+
Provides-Extra: dev
|
|
32
|
+
Requires-Dist: tox==4.12.1; extra == "dev"
|
|
33
|
+
Requires-Dist: pluggy; extra == "dev"
|
|
34
|
+
Requires-Dist: dj-database-url==2.1.0; extra == "dev"
|
|
35
|
+
Requires-Dist: django-braces==1.15.0; extra == "dev"
|
|
36
|
+
Requires-Dist: tblib; extra == "dev"
|
|
37
|
+
Requires-Dist: filelock; extra == "dev"
|
|
38
|
+
Requires-Dist: coverage; extra == "dev"
|
|
39
|
+
Requires-Dist: PyYAML; extra == "dev"
|
|
40
|
+
Requires-Dist: sphinx==1.8.1; extra == "dev"
|
|
41
|
+
Requires-Dist: Jinja2<2.11,>=2.3; extra == "dev"
|
|
42
|
+
Requires-Dist: sphinx_rtd_theme==0.4.2; extra == "dev"
|
|
43
|
+
Requires-Dist: MarkupSafe<2.1.0,>=0.23; extra == "dev"
|
|
44
|
+
Dynamic: author
|
|
45
|
+
Dynamic: author-email
|
|
46
|
+
Dynamic: classifier
|
|
47
|
+
Dynamic: home-page
|
|
48
|
+
Dynamic: license
|
|
49
|
+
Dynamic: license-file
|
|
50
|
+
Dynamic: provides-extra
|
|
51
|
+
Dynamic: requires-dist
|
|
52
|
+
Dynamic: summary
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
DjanQuiltDB - Django Database Sharding
|
|
2
|
+
======================================
|
|
3
|
+
|
|
4
|
+
**DjanQuiltDB** is an extension to the Django web framework that provides helper functions to split a database based on
|
|
5
|
+
top level hierarchy. It is specifically designed to support horizontal sharding not just within a single database
|
|
6
|
+
cluster, but also across multiple database clusters, thus allowing you to scale database capacity both vertically and
|
|
7
|
+
horizontally as your dataset grows.
|
|
8
|
+
|
|
9
|
+
This library attempts to combine the best of as many worlds as possible. Tenant-specific data is kept in tenant-specific
|
|
10
|
+
PostgreSQL schemas, while data that is tenant-agnostic or shared can be kept in public schemas, so as to deduplicate.
|
|
11
|
+
There are helpers to split data off from one shard to another, to migrate a shard across nodes, to synchronize changes
|
|
12
|
+
across nodes, etc.
|
|
13
|
+
|
|
14
|
+
As the name suggests, this approach provides an interface to data, that may in reality be scattered across various
|
|
15
|
+
schemas in various database clusters, and presents it as a coherent and easily accessible patchwork of tables, resulting
|
|
16
|
+
in a database resembling a quilt.
|
|
17
|
+
|
|
18
|
+
Development
|
|
19
|
+
===========
|
|
20
|
+
|
|
21
|
+
To setup your development environment it is important to install the development requirements first::
|
|
22
|
+
|
|
23
|
+
pip install -e .[dev]
|
|
24
|
+
|
|
25
|
+
Next, copy the example secrets file in the ``djanquiltdb`` directory and adjust the parameters::
|
|
26
|
+
|
|
27
|
+
cp secrets.json.example secrets.json
|
|
28
|
+
|
|
29
|
+
Tests
|
|
30
|
+
-----
|
|
31
|
+
|
|
32
|
+
Tox can be used to run the test suite against multiple Python and Django versions::
|
|
33
|
+
|
|
34
|
+
tox
|
|
35
|
+
|
|
36
|
+
Building
|
|
37
|
+
--------
|
|
38
|
+
|
|
39
|
+
To build the library, simply run::
|
|
40
|
+
|
|
41
|
+
python setup.py build
|
|
42
|
+
|
|
43
|
+
And to make a distribution, run ::
|
|
44
|
+
|
|
45
|
+
python setup.py sdist
|
|
46
|
+
|
|
47
|
+
The result is then bound in the /dist folder
|
|
48
|
+
|
|
49
|
+
Documentation
|
|
50
|
+
-------------
|
|
51
|
+
|
|
52
|
+
Documentation can be found in the `/docs` directory. Build the documentation with::
|
|
53
|
+
|
|
54
|
+
python setup.py build_sphinx
|
|
55
|
+
|
|
56
|
+
Attribution
|
|
57
|
+
===========
|
|
58
|
+
|
|
59
|
+
This is an independently maintained fork of the patchman-django-sharding library originally created and maintained by
|
|
60
|
+
Patchman B.V. (2017-2023) and Cloud Linux Software, Inc. (2023-2025).
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
from enum import Enum
|
|
2
|
+
|
|
3
|
+
__version__ = '3.0.0'
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
default_app_config = 'djanquiltdb.apps.DjanQuiltDBConfig'
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class ShardingMode(Enum):
|
|
10
|
+
MIRRORED = 'M'
|
|
11
|
+
PUBLIC = 'P'
|
|
12
|
+
SHARDED = 'S'
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
public_modes = (ShardingMode.MIRRORED, ShardingMode.PUBLIC)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class State(object):
|
|
19
|
+
ACTIVE = 'A'
|
|
20
|
+
MAINTENANCE = 'M'
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
STATES = (
|
|
24
|
+
(State.ACTIVE, 'Active'),
|
|
25
|
+
(State.MAINTENANCE, 'Maintenance'),
|
|
26
|
+
)
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
import functools
|
|
2
|
+
import inspect
|
|
3
|
+
import types
|
|
4
|
+
|
|
5
|
+
from django.apps import AppConfig
|
|
6
|
+
from django.conf import settings
|
|
7
|
+
from django.contrib.auth import get_user_model
|
|
8
|
+
from django.core.exceptions import ImproperlyConfigured
|
|
9
|
+
from django.utils.module_loading import import_string
|
|
10
|
+
|
|
11
|
+
from djanquiltdb import ShardingMode
|
|
12
|
+
from djanquiltdb.db import connection
|
|
13
|
+
from djanquiltdb.decorators import class_method_use_shard, class_method_use_shard_from_db_arg
|
|
14
|
+
from djanquiltdb.options import ShardOptions
|
|
15
|
+
from djanquiltdb.postgresql_backend.base import ShardDatabaseWrapper
|
|
16
|
+
from djanquiltdb.utils import get_all_mirrored_models, get_all_public_models, get_all_sharded_models
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class DjanQuiltDBConfig(AppConfig):
|
|
20
|
+
name = 'djanquiltdb'
|
|
21
|
+
verbose_name = 'DjanQuiltDB'
|
|
22
|
+
|
|
23
|
+
def ready(self):
|
|
24
|
+
from .models import BaseShard
|
|
25
|
+
|
|
26
|
+
if 'QUILT_DB' not in dir(settings) or not isinstance(settings.QUILT_DB, dict):
|
|
27
|
+
raise ImproperlyConfigured('Missing or incorrect type of a setting QUILT_DB.')
|
|
28
|
+
|
|
29
|
+
# Validate shard and node class settings
|
|
30
|
+
if 'SHARD_CLASS' not in settings.QUILT_DB:
|
|
31
|
+
raise ImproperlyConfigured('Missing or incorrect type of a setting QUILT_DB["{}"].'.format('SHARD_CLASS'))
|
|
32
|
+
class_ = import_string(settings.QUILT_DB['SHARD_CLASS'])
|
|
33
|
+
if not issubclass(class_, BaseShard):
|
|
34
|
+
raise ImproperlyConfigured(
|
|
35
|
+
'The type {} should inherit from {}.'.format(settings.QUILT_DB['SHARD_CLASS'], BaseShard.__name__)
|
|
36
|
+
)
|
|
37
|
+
if hasattr(class_, '__sharding_mode') and getattr(class_, '__sharding_mode') == ShardingMode.SHARDED:
|
|
38
|
+
raise ImproperlyConfigured(
|
|
39
|
+
'The Shard model cannot itself be sharded. It can only be non-sharded or mirrored.'
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
override_sharding_mode = settings.QUILT_DB.setdefault('OVERRIDE_SHARDING_MODE', {})
|
|
43
|
+
if not isinstance(override_sharding_mode, dict):
|
|
44
|
+
raise ImproperlyConfigured("Incorrect setting value of QUILT_DB['OVERRIDE_SHARDING_MODE'].")
|
|
45
|
+
|
|
46
|
+
for key, value in override_sharding_mode.items():
|
|
47
|
+
_validate_override_sharding_mode_entry(key, value)
|
|
48
|
+
|
|
49
|
+
# Convert app and model names to lowercase
|
|
50
|
+
settings.QUILT_DB['OVERRIDE_SHARDING_MODE'] = dict(
|
|
51
|
+
(tuple(x.lower() for x in k), v) for k, v in override_sharding_mode.items()
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
if (
|
|
55
|
+
'DATABASE_ROUTERS' not in dir(settings)
|
|
56
|
+
or 'djanquiltdb.router.DynamicDbRouter' not in settings.DATABASE_ROUTERS
|
|
57
|
+
):
|
|
58
|
+
raise ImproperlyConfigured(
|
|
59
|
+
'djanquiltdb.router.DynamicDbRouter must be present in the DATABASE_ROUTERS setting.'
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
if (
|
|
63
|
+
'SESSION_ENGINE' in dir(settings)
|
|
64
|
+
and settings.SESSION_ENGINE == 'django.contrib.sessions.backends.cached_db'
|
|
65
|
+
and getattr(get_user_model(), '__sharding_mode', False) in [ShardingMode.MIRRORED, ShardingMode.SHARDED]
|
|
66
|
+
):
|
|
67
|
+
raise ImproperlyConfigured(
|
|
68
|
+
'When the user model is sharded, you cannot use django.contrib.sessions.backends.cached_db '
|
|
69
|
+
"to store sessions. It references the user table and won't know where to find it."
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
if (
|
|
73
|
+
'PRIMARY_DB_ALIAS' not in settings.QUILT_DB or not settings.QUILT_DB.get('PRIMARY_DB_ALIAS', None)
|
|
74
|
+
) and get_all_mirrored_models():
|
|
75
|
+
raise ImproperlyConfigured("There are MIRRORED models, but QUILT_DB['PRIMARY_DB_ALIAS'] is not set.")
|
|
76
|
+
|
|
77
|
+
_validate_public_models()
|
|
78
|
+
_patch_connections()
|
|
79
|
+
_patch_transactions()
|
|
80
|
+
_initialize_sharded_models()
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _validate_public_models():
|
|
84
|
+
for model in get_all_public_models():
|
|
85
|
+
if not hasattr(model._meta, 'unique_together') or not model._meta.unique_together:
|
|
86
|
+
raise ImproperlyConfigured(
|
|
87
|
+
f'{model._meta.app_label}.{model.__name__} must have "unique_together" meta attribute set.'
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
if not hasattr(model, 'natural_key'):
|
|
91
|
+
raise ImproperlyConfigured(f'{model._meta.app_label}.{model.__name__} must define "natural_key" method.')
|
|
92
|
+
|
|
93
|
+
if not hasattr(model._default_manager, 'get_by_natural_key'):
|
|
94
|
+
raise ImproperlyConfigured(
|
|
95
|
+
f'The default manager for {model._meta.app_label}.{model.__name__} must'
|
|
96
|
+
'define "get_by_natural_key" method.'
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _validate_override_sharding_mode_entry(key, value):
|
|
101
|
+
if not (isinstance(key, tuple) and len(key) in (1, 2) and isinstance(value, ShardingMode)):
|
|
102
|
+
raise ImproperlyConfigured(
|
|
103
|
+
'The override sharding mode entry is improperly configured: {{ {}: {} }}'.format(repr(key), repr(value))
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _initialize_sharded_models():
|
|
108
|
+
"""
|
|
109
|
+
Initialize sharded models by overriding all methods to add a use_shard context manager that makes sure all queries
|
|
110
|
+
are done in that same shard as the object is living in.
|
|
111
|
+
"""
|
|
112
|
+
from_db_functions = {model: model.from_db for model in get_all_sharded_models(include_proxy=True)}
|
|
113
|
+
|
|
114
|
+
for model in get_all_sharded_models(include_proxy=True):
|
|
115
|
+
for attr, func in inspect.getmembers(model, inspect.isfunction):
|
|
116
|
+
# getattr(model, attr) will trigger dynamic lookup via the descriptor protocol, __getattr__ or
|
|
117
|
+
# __getattribute__. Therefore, we use inspect.getattr_static to strip out staticmethods (which we don't want
|
|
118
|
+
# to decorate).
|
|
119
|
+
if isinstance(inspect.getattr_static(model, attr), types.FunctionType):
|
|
120
|
+
# And decorate all model methods so that the methods will all run in the same shard context as the
|
|
121
|
+
# instance is living in
|
|
122
|
+
setattr(model, attr, class_method_use_shard(func))
|
|
123
|
+
|
|
124
|
+
# Setting the from_db function for a Model that is the parent of a ProxyModel will corrupt the ProxyModels
|
|
125
|
+
# version of the same function. So we have saved the original from_db function at the start, and use that.
|
|
126
|
+
model.add_to_class('from_db', class_method_use_shard_from_db_arg(from_db_functions[model]))
|
|
127
|
+
|
|
128
|
+
_initialize_sharded_model_querysets(model)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def post_init(func):
|
|
132
|
+
@functools.wraps(func)
|
|
133
|
+
def inner(self, *args, hints=None, **kwargs):
|
|
134
|
+
hints = hints or {}
|
|
135
|
+
|
|
136
|
+
if '_shard_options' not in hints and not connection.is_public_schema():
|
|
137
|
+
hints['_shard_options'] = connection.shard_options
|
|
138
|
+
|
|
139
|
+
func(self, *args, hints=hints, **kwargs)
|
|
140
|
+
|
|
141
|
+
inner.__decorator__ = post_init
|
|
142
|
+
return inner
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _initialize_sharded_model_querysets(model):
|
|
146
|
+
"""
|
|
147
|
+
Override all the querysets of the managers, so they can remember the shard where they are initialized on
|
|
148
|
+
"""
|
|
149
|
+
for instance in model._meta.managers:
|
|
150
|
+
if (
|
|
151
|
+
hasattr(instance._queryset_class.__init__, '__decorator__')
|
|
152
|
+
and instance._queryset_class.__init__.__decorator__ == post_init
|
|
153
|
+
):
|
|
154
|
+
continue
|
|
155
|
+
setattr(instance._queryset_class, '__init__', post_init(instance._queryset_class.__init__))
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def patch_getitem(func):
|
|
159
|
+
@functools.wraps(func)
|
|
160
|
+
def inner(self, alias):
|
|
161
|
+
# If we're planning to just go into the public schema, then we're going to use the normal connection for that
|
|
162
|
+
if isinstance(alias, str) and '|' not in alias or isinstance(alias, ShardOptions) and alias.is_public_schema():
|
|
163
|
+
return func(self, alias if isinstance(alias, str) else alias.node_name)
|
|
164
|
+
|
|
165
|
+
options = ShardOptions.from_alias(alias)
|
|
166
|
+
|
|
167
|
+
# Retrieves the main connection to the database, so we can still do connection pooling
|
|
168
|
+
connection_ = func(self, options.node_name)
|
|
169
|
+
|
|
170
|
+
# Sets up the sharded connection
|
|
171
|
+
return ShardDatabaseWrapper(connection_, options)
|
|
172
|
+
|
|
173
|
+
return inner
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _patch_connections():
|
|
177
|
+
"""
|
|
178
|
+
Monkeypatch django.db.connection and django.db.connections to accept:
|
|
179
|
+
|
|
180
|
+
* node_name
|
|
181
|
+
* node_name|schema_name
|
|
182
|
+
* tuple (node_name, schema_name)
|
|
183
|
+
* ShardOptions instance
|
|
184
|
+
* Shard model instance
|
|
185
|
+
"""
|
|
186
|
+
import django.db
|
|
187
|
+
from django.db.utils import ConnectionHandler
|
|
188
|
+
|
|
189
|
+
setattr(ConnectionHandler, '__getitem__', patch_getitem(ConnectionHandler.__getitem__))
|
|
190
|
+
setattr(django.db, 'connection', connection)
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _patch_transactions():
|
|
194
|
+
"""
|
|
195
|
+
Monkeypatch django.db.transaction.get_connection to use the active node, not always the default:
|
|
196
|
+
"""
|
|
197
|
+
import django.db.transaction
|
|
198
|
+
|
|
199
|
+
from djanquiltdb.transaction import atomic, get_connection
|
|
200
|
+
|
|
201
|
+
setattr(django.db.transaction, 'get_connection', get_connection)
|
|
202
|
+
setattr(django.db.transaction, 'atomic', atomic)
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import progressbar
|
|
2
|
+
from django.db.models.deletion import Collector
|
|
3
|
+
from django.db.models.deletion import get_candidate_relations_to_delete as get_candidate_relations_to_collect
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class SimpleCollector(Collector):
|
|
7
|
+
"""
|
|
8
|
+
Simple collector does basically the same as the default Django collector, but also follows the on delete SET NULL
|
|
9
|
+
relations. We need this to move data from one shard to another.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
def __init__(self, connection, verbose=False):
|
|
13
|
+
super().__init__(using=connection.alias)
|
|
14
|
+
|
|
15
|
+
self.connection = connection
|
|
16
|
+
self.verbose = verbose
|
|
17
|
+
self.data_points = 0
|
|
18
|
+
|
|
19
|
+
if self.verbose:
|
|
20
|
+
self.bar = progressbar.ProgressBar(
|
|
21
|
+
max_value=progressbar.UnknownLength,
|
|
22
|
+
widgets=[
|
|
23
|
+
progressbar.RotatingMarker(),
|
|
24
|
+
' Collected ',
|
|
25
|
+
progressbar.Counter(),
|
|
26
|
+
' datapoints; ',
|
|
27
|
+
progressbar.Timer(),
|
|
28
|
+
],
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
def add(self, *args, **kwargs):
|
|
32
|
+
new_objs = super().add(*args, **kwargs)
|
|
33
|
+
|
|
34
|
+
if self.verbose:
|
|
35
|
+
self.data_points += len(new_objs)
|
|
36
|
+
self.bar.update(self.data_points)
|
|
37
|
+
|
|
38
|
+
return new_objs
|
|
39
|
+
|
|
40
|
+
def get_batches(self, objs, field):
|
|
41
|
+
"""
|
|
42
|
+
Returns the objs in suitably sized batches for the used connection.
|
|
43
|
+
"""
|
|
44
|
+
conn_batch_size = max(self.connection.ops.bulk_batch_size([field.name], objs), 1)
|
|
45
|
+
if len(objs) > conn_batch_size:
|
|
46
|
+
return [objs[i : i + conn_batch_size] for i in range(0, len(objs), conn_batch_size)]
|
|
47
|
+
else:
|
|
48
|
+
return [objs]
|
|
49
|
+
|
|
50
|
+
def finish_bar(self):
|
|
51
|
+
if self.verbose:
|
|
52
|
+
self.bar.finish()
|
|
53
|
+
|
|
54
|
+
def collect(
|
|
55
|
+
self,
|
|
56
|
+
objs,
|
|
57
|
+
source=None,
|
|
58
|
+
nullable=False,
|
|
59
|
+
collect_related=True,
|
|
60
|
+
source_attr=None,
|
|
61
|
+
reverse_dependency=False,
|
|
62
|
+
keep_parents=False,
|
|
63
|
+
fail_on_restricted=True,
|
|
64
|
+
):
|
|
65
|
+
"""
|
|
66
|
+
Kind of the same as the original collector, but now also follows the on delete SET NULL relations.
|
|
67
|
+
"""
|
|
68
|
+
new_objs = self.add(objs, source, nullable, reverse_dependency=reverse_dependency)
|
|
69
|
+
|
|
70
|
+
if not new_objs:
|
|
71
|
+
self.finish_bar()
|
|
72
|
+
return
|
|
73
|
+
|
|
74
|
+
model = new_objs[0].__class__
|
|
75
|
+
|
|
76
|
+
# Recursively collect concrete model's parent models, but not their
|
|
77
|
+
# related objects. These will be found by meta.get_fields()
|
|
78
|
+
concrete_model = model._meta.concrete_model
|
|
79
|
+
for ptr in concrete_model._meta.parents.values():
|
|
80
|
+
if ptr:
|
|
81
|
+
# FIXME: This seems to be buggy and execute a query for each
|
|
82
|
+
# parent object fetch. We have the parent data in the obj,
|
|
83
|
+
# but we don't have a nice way to turn that data into parent
|
|
84
|
+
# object instance.
|
|
85
|
+
parent_objs = [getattr(obj, ptr.name) for obj in new_objs]
|
|
86
|
+
self.collect(
|
|
87
|
+
parent_objs, source=model, source_attr=ptr.remote_field.related_name, collect_related=False
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
if collect_related:
|
|
91
|
+
for related in get_candidate_relations_to_collect(model._meta):
|
|
92
|
+
field = related.field
|
|
93
|
+
batches = self.get_batches(new_objs, field)
|
|
94
|
+
for batch in batches:
|
|
95
|
+
sub_objs = self.related_objects(related.related_model, [field], batch)
|
|
96
|
+
if not sub_objs:
|
|
97
|
+
continue
|
|
98
|
+
self.collect(sub_objs, source=model, source_attr=field.name)
|
|
99
|
+
for field in model._meta.private_fields:
|
|
100
|
+
if hasattr(field, 'bulk_related_objects'):
|
|
101
|
+
# It's something like generic foreign key.
|
|
102
|
+
sub_objs = field.bulk_related_objects(new_objs, self.using)
|
|
103
|
+
self.collect(sub_objs, source=model, nullable=True, fail_on_restricted=False)
|
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
from typing import TYPE_CHECKING, Optional
|
|
2
|
+
|
|
3
|
+
from django.apps import AppConfig
|
|
4
|
+
from django.core.exceptions import ImproperlyConfigured
|
|
5
|
+
|
|
6
|
+
if TYPE_CHECKING:
|
|
7
|
+
from djanquiltdb.contrib.quilt_admin.shard_selector import BaseAdminShardSelector
|
|
8
|
+
|
|
9
|
+
ADMIN_SHARD_SELECTOR_CLASS: Optional[BaseAdminShardSelector] = None
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class ShardedAdminAppConfig(AppConfig):
|
|
13
|
+
name = 'djanquiltdb.contrib.quilt_admin'
|
|
14
|
+
verbose_name = 'DjanQuiltDB Admin Integration'
|
|
15
|
+
|
|
16
|
+
def ready(self):
|
|
17
|
+
from django.conf import settings
|
|
18
|
+
from django.utils.module_loading import import_string
|
|
19
|
+
|
|
20
|
+
from djanquiltdb.contrib.quilt_admin.shard_selector import BaseAdminShardSelector
|
|
21
|
+
|
|
22
|
+
global ADMIN_SHARD_SELECTOR_CLASS
|
|
23
|
+
|
|
24
|
+
if hasattr(settings, 'QUILT_ADMIN') and 'SHARD_SELECTOR_CLASS' in settings.QUILT_ADMIN:
|
|
25
|
+
shard_selector_class = settings.QUILT_ADMIN['SHARD_SELECTOR_CLASS']
|
|
26
|
+
else:
|
|
27
|
+
shard_selector_class = 'djanquiltdb.contrib.quilt_admin.shard_selector.AdminShardSelector'
|
|
28
|
+
|
|
29
|
+
ADMIN_SHARD_SELECTOR_CLASS = import_string(shard_selector_class)
|
|
30
|
+
|
|
31
|
+
if not issubclass(ADMIN_SHARD_SELECTOR_CLASS, BaseAdminShardSelector):
|
|
32
|
+
raise ImproperlyConfigured(
|
|
33
|
+
f"Shard selector class {ADMIN_SHARD_SELECTOR_CLASS} doesn't subclass BaseAdminShardSelector"
|
|
34
|
+
)
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
from djanquiltdb.contrib.quilt_admin.apps import ADMIN_SHARD_SELECTOR_CLASS
|
|
2
|
+
from djanquiltdb.utils import get_mapping_class, get_shard_class
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def admin_shard_context(request):
|
|
6
|
+
"""
|
|
7
|
+
Context processor to provide shard information to admin templates.
|
|
8
|
+
Adds shard switcher context to admin templates.
|
|
9
|
+
|
|
10
|
+
If a mapping model is configured (QUILT_DB["MAPPING_MODEL"]), the selector lists
|
|
11
|
+
mapping entries (by mapping value) instead of shards (by shard id).
|
|
12
|
+
"""
|
|
13
|
+
context = {
|
|
14
|
+
'available_shards': [],
|
|
15
|
+
'current_shard_id': None, # kept for backwards compatibility
|
|
16
|
+
'shard_switcher_options': [],
|
|
17
|
+
'shard_switcher_field_name': 'shard_id',
|
|
18
|
+
'current_shard_switcher_value': '',
|
|
19
|
+
'shard_switcher_mode': 'shard', # or 'mapping'
|
|
20
|
+
'shard_maintenance_mode': False,
|
|
21
|
+
'shard_maintenance_message': None,
|
|
22
|
+
}
|
|
23
|
+
# Add maintenance status if available
|
|
24
|
+
if hasattr(request, '_shard_maintenance_mode'):
|
|
25
|
+
context['shard_maintenance_mode'] = request._shard_maintenance_mode
|
|
26
|
+
context['shard_maintenance_message'] = getattr(request, '_shard_maintenance_message', None)
|
|
27
|
+
# Only add shard context if user is authenticated and in admin
|
|
28
|
+
if hasattr(request, 'user') and request.user.is_authenticated and request.path.startswith('/admin/'):
|
|
29
|
+
try:
|
|
30
|
+
mapping_class = get_mapping_class()
|
|
31
|
+
if mapping_class:
|
|
32
|
+
mapping_field = getattr(mapping_class, 'mapping_field', None)
|
|
33
|
+
|
|
34
|
+
qs = mapping_class.objects.using('default').select_related('shard').all()
|
|
35
|
+
if mapping_field:
|
|
36
|
+
qs = qs.order_by(mapping_field)
|
|
37
|
+
else:
|
|
38
|
+
qs = qs.order_by('pk')
|
|
39
|
+
|
|
40
|
+
mappings = list(qs)
|
|
41
|
+
context['available_shards'] = mappings
|
|
42
|
+
context['shard_switcher_mode'] = 'mapping'
|
|
43
|
+
context['shard_switcher_field_name'] = 'mapping_value'
|
|
44
|
+
|
|
45
|
+
current_value = ADMIN_SHARD_SELECTOR_CLASS.retrieve_override_value(request)
|
|
46
|
+
context['current_shard_switcher_value'] = '' if current_value is None else str(current_value)
|
|
47
|
+
|
|
48
|
+
options = []
|
|
49
|
+
for mapping in mappings:
|
|
50
|
+
if mapping_field:
|
|
51
|
+
mapping_value = getattr(mapping, mapping_field)
|
|
52
|
+
mapping_value_str = '' if mapping_value is None else str(mapping_value)
|
|
53
|
+
else:
|
|
54
|
+
mapping_value = mapping.pk
|
|
55
|
+
mapping_value_str = str(mapping.pk)
|
|
56
|
+
|
|
57
|
+
shard = getattr(mapping, 'shard', None)
|
|
58
|
+
label = ADMIN_SHARD_SELECTOR_CLASS.format_override_option(mapping_value, shard, mapping)
|
|
59
|
+
|
|
60
|
+
options.append({'value': mapping_value_str, 'label': label})
|
|
61
|
+
|
|
62
|
+
context['shard_switcher_options'] = options
|
|
63
|
+
else:
|
|
64
|
+
shard_class = get_shard_class()
|
|
65
|
+
# Get all shards from the primary database
|
|
66
|
+
# Using .using('default') to ensure we query from the primary database
|
|
67
|
+
shards = list(shard_class.objects.using('default').all().order_by('alias'))
|
|
68
|
+
context['available_shards'] = shards
|
|
69
|
+
|
|
70
|
+
current_id = ADMIN_SHARD_SELECTOR_CLASS.retrieve_override_value(request)
|
|
71
|
+
context['current_shard_id'] = current_id
|
|
72
|
+
context['current_shard_switcher_value'] = '' if current_id is None else str(current_id)
|
|
73
|
+
|
|
74
|
+
context['shard_switcher_options'] = [
|
|
75
|
+
{
|
|
76
|
+
'value': str(shard.id),
|
|
77
|
+
'label': ADMIN_SHARD_SELECTOR_CLASS.format_override_option(shard.id, shard),
|
|
78
|
+
}
|
|
79
|
+
for shard in shards
|
|
80
|
+
]
|
|
81
|
+
except Exception:
|
|
82
|
+
# If there's any error (e.g., database not ready, no shards exist), just return empty context
|
|
83
|
+
pass
|
|
84
|
+
|
|
85
|
+
return context
|