djanquiltdb 3.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. djanquiltdb-3.0.0/LICENSE +28 -0
  2. djanquiltdb-3.0.0/MANIFEST.in +7 -0
  3. djanquiltdb-3.0.0/PKG-INFO +52 -0
  4. djanquiltdb-3.0.0/README.rst +60 -0
  5. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/__init__.py +26 -0
  6. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/apps.py +202 -0
  7. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/collector.py +103 -0
  8. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/contrib/__init__.py +0 -0
  9. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/contrib/quilt_admin/__init__.py +0 -0
  10. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/contrib/quilt_admin/apps.py +34 -0
  11. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/contrib/quilt_admin/context_processors.py +85 -0
  12. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/contrib/quilt_admin/middleware.py +196 -0
  13. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/contrib/quilt_admin/shard_selector.py +78 -0
  14. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/contrib/quilt_admin/templates/admin/base_site.html +72 -0
  15. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/contrib/quilt_admin/urls.py +7 -0
  16. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/contrib/quilt_admin/utils.py +27 -0
  17. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/contrib/quilt_admin/views.py +94 -0
  18. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/contrib/quilt_auth/__init__.py +1 -0
  19. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/contrib/quilt_auth/apps.py +6 -0
  20. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/contrib/quilt_auth/management/__init__.py +0 -0
  21. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/contrib/quilt_auth/management/commands/__init__.py +0 -0
  22. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/contrib/quilt_auth/management/commands/createsuperuser.py +119 -0
  23. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/db/__init__.py +31 -0
  24. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/decorators.py +409 -0
  25. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/exceptions.py +2 -0
  26. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/forms.py +18 -0
  27. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/management/__init__.py +0 -0
  28. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/management/base.py +39 -0
  29. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/management/commands/__init__.py +0 -0
  30. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/management/commands/flush.py +94 -0
  31. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/management/commands/loaddata.py +308 -0
  32. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/management/commands/migrate.py +15 -0
  33. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/management/commands/migrate_shards.py +387 -0
  34. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/management/commands/move_data_to_shard.py +522 -0
  35. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/management/commands/move_shard_to_node.py +535 -0
  36. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/management/commands/move_sharded_models.py +182 -0
  37. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/management/commands/purge_schema.py +117 -0
  38. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/management/commands/purge_shard_data.py +196 -0
  39. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/management/commands/sqlflush.py +62 -0
  40. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/middleware.py +168 -0
  41. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/models.py +117 -0
  42. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/options.py +103 -0
  43. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/postgresql_backend/__init__.py +1 -0
  44. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/postgresql_backend/base.py +712 -0
  45. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/postgresql_backend/creation.py +74 -0
  46. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/postgresql_backend/introspection.py +294 -0
  47. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/postgresql_backend/operations.py +45 -0
  48. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/postgresql_backend/utils.py +73 -0
  49. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/router.py +123 -0
  50. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/sessions.py +128 -0
  51. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/transaction.py +48 -0
  52. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb/utils.py +717 -0
  53. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb.egg-info/PKG-INFO +52 -0
  54. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb.egg-info/SOURCES.txt +63 -0
  55. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb.egg-info/dependency_links.txt +1 -0
  56. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb.egg-info/requires.txt +27 -0
  57. djanquiltdb-3.0.0/djanquiltdb/djanquiltdb.egg-info/top_level.txt +1 -0
  58. djanquiltdb-3.0.0/djanquiltdb/templates/example/connection_exception.html +6 -0
  59. djanquiltdb-3.0.0/djanquiltdb/templates/example/home.html +11 -0
  60. djanquiltdb-3.0.0/djanquiltdb/templates/example/login.html +10 -0
  61. djanquiltdb-3.0.0/djanquiltdb/templates/example/state_exception.html +6 -0
  62. djanquiltdb-3.0.0/pyproject.toml +15 -0
  63. djanquiltdb-3.0.0/setup.cfg +7 -0
  64. djanquiltdb-3.0.0/setup.py +63 -0
@@ -0,0 +1,28 @@
1
+ Copyright (c) 2017-2023, Patchman B.V.
2
+ Copyright (c) 2023-2025, Cloud Linux Software, Inc.
3
+ Copyright (c) 2025-2026, DjanQuiltDB Project
4
+
5
+ All rights reserved.
6
+
7
+ Redistribution and use in source and binary forms, with or without
8
+ modification, are permitted provided that the following conditions are met:
9
+ * Redistributions of source code must retain the above copyright
10
+ notice, this list of conditions and the following disclaimer.
11
+ * Redistributions in binary form must reproduce the above copyright
12
+ notice, this list of conditions and the following disclaimer in the
13
+ documentation and/or other materials provided with the distribution.
14
+ * Neither the names of Patchman B.V., Cloud Linux Software, Inc. or
15
+ DjanQuiltDB, nor the names of their contributors may be used to
16
+ endorse or promote products derived from this software without specific
17
+ prior written permission.
18
+
19
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
20
+ ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
21
+ WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22
+ DISCLAIMED. IN NO EVENT SHALL PATCHMAN B.V., CLOUD LINUX SOFTWARE, INC., OR
23
+ THE DJANQUILTDB PROJECT BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
24
+ EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
25
+ OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26
+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
27
+ STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28
+ OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
@@ -0,0 +1,7 @@
1
+ include LICENSE
2
+ include README.rst
3
+ include MANIFEST.in
4
+ exclude .DS_Store
5
+
6
+ # Django template files (including contrib apps like quilt_admin)
7
+ recursive-include djanquiltdb *.html
@@ -0,0 +1,52 @@
1
+ Metadata-Version: 2.4
2
+ Name: djanquiltdb
3
+ Version: 3.0.0
4
+ Summary: Library to shard a database on the hierarchy's top level table.
5
+ Home-page: https://www.github.com/DjanQuiltDB/djanquiltdb
6
+ Author: DjanQuiltDB Project; Cloud Linux Software, Inc.; Patchman B.V.
7
+ Author-email: djanquiltdb@portal42.net; info@cloudlinux.com; hello@patchman.co
8
+ License: BSD-3-Clause
9
+ Classifier: Development Status :: 5 - Production/Stable
10
+ Classifier: Environment :: Web Environment
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: Operating System :: OS Independent
13
+ Classifier: Programming Language :: Python
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.14
16
+ Classifier: Framework :: Django
17
+ Classifier: Framework :: Django :: 6.0
18
+ License-File: LICENSE
19
+ Requires-Dist: django<7.0,>=6.0
20
+ Requires-Dist: psycopg[binary]>=3.0.0
21
+ Requires-Dist: progressbar2
22
+ Provides-Extra: test
23
+ Requires-Dist: tox==4.12.1; extra == "test"
24
+ Requires-Dist: pluggy; extra == "test"
25
+ Requires-Dist: dj-database-url==2.1.0; extra == "test"
26
+ Requires-Dist: django-braces==1.15.0; extra == "test"
27
+ Requires-Dist: tblib; extra == "test"
28
+ Requires-Dist: filelock; extra == "test"
29
+ Requires-Dist: coverage; extra == "test"
30
+ Requires-Dist: PyYAML; extra == "test"
31
+ Provides-Extra: dev
32
+ Requires-Dist: tox==4.12.1; extra == "dev"
33
+ Requires-Dist: pluggy; extra == "dev"
34
+ Requires-Dist: dj-database-url==2.1.0; extra == "dev"
35
+ Requires-Dist: django-braces==1.15.0; extra == "dev"
36
+ Requires-Dist: tblib; extra == "dev"
37
+ Requires-Dist: filelock; extra == "dev"
38
+ Requires-Dist: coverage; extra == "dev"
39
+ Requires-Dist: PyYAML; extra == "dev"
40
+ Requires-Dist: sphinx==1.8.1; extra == "dev"
41
+ Requires-Dist: Jinja2<2.11,>=2.3; extra == "dev"
42
+ Requires-Dist: sphinx_rtd_theme==0.4.2; extra == "dev"
43
+ Requires-Dist: MarkupSafe<2.1.0,>=0.23; extra == "dev"
44
+ Dynamic: author
45
+ Dynamic: author-email
46
+ Dynamic: classifier
47
+ Dynamic: home-page
48
+ Dynamic: license
49
+ Dynamic: license-file
50
+ Dynamic: provides-extra
51
+ Dynamic: requires-dist
52
+ Dynamic: summary
@@ -0,0 +1,60 @@
1
+ DjanQuiltDB - Django Database Sharding
2
+ ======================================
3
+
4
+ **DjanQuiltDB** is an extension to the Django web framework that provides helper functions to split a database based on
5
+ top level hierarchy. It is specifically designed to support horizontal sharding not just within a single database
6
+ cluster, but also across multiple database clusters, thus allowing you to scale database capacity both vertically and
7
+ horizontally as your dataset grows.
8
+
9
+ This library attempts to combine the best of as many worlds as possible. Tenant-specific data is kept in tenant-specific
10
+ PostgreSQL schemas, while data that is tenant-agnostic or shared can be kept in public schemas, so as to deduplicate.
11
+ There are helpers to split data off from one shard to another, to migrate a shard across nodes, to synchronize changes
12
+ across nodes, etc.
13
+
14
+ As the name suggests, this approach provides an interface to data, that may in reality be scattered across various
15
+ schemas in various database clusters, and presents it as a coherent and easily accessible patchwork of tables, resulting
16
+ in a database resembling a quilt.
17
+
18
+ Development
19
+ ===========
20
+
21
+ To setup your development environment it is important to install the development requirements first::
22
+
23
+ pip install -e .[dev]
24
+
25
+ Next, copy the example secrets file in the ``djanquiltdb`` directory and adjust the parameters::
26
+
27
+ cp secrets.json.example secrets.json
28
+
29
+ Tests
30
+ -----
31
+
32
+ Tox can be used to run the test suite against multiple Python and Django versions::
33
+
34
+ tox
35
+
36
+ Building
37
+ --------
38
+
39
+ To build the library, simply run::
40
+
41
+ python setup.py build
42
+
43
+ And to make a distribution, run ::
44
+
45
+ python setup.py sdist
46
+
47
+ The result is then bound in the /dist folder
48
+
49
+ Documentation
50
+ -------------
51
+
52
+ Documentation can be found in the `/docs` directory. Build the documentation with::
53
+
54
+ python setup.py build_sphinx
55
+
56
+ Attribution
57
+ ===========
58
+
59
+ This is an independently maintained fork of the patchman-django-sharding library originally created and maintained by
60
+ Patchman B.V. (2017-2023) and Cloud Linux Software, Inc. (2023-2025).
@@ -0,0 +1,26 @@
1
+ from enum import Enum
2
+
3
+ __version__ = '3.0.0'
4
+
5
+
6
+ default_app_config = 'djanquiltdb.apps.DjanQuiltDBConfig'
7
+
8
+
9
+ class ShardingMode(Enum):
10
+ MIRRORED = 'M'
11
+ PUBLIC = 'P'
12
+ SHARDED = 'S'
13
+
14
+
15
+ public_modes = (ShardingMode.MIRRORED, ShardingMode.PUBLIC)
16
+
17
+
18
+ class State(object):
19
+ ACTIVE = 'A'
20
+ MAINTENANCE = 'M'
21
+
22
+
23
+ STATES = (
24
+ (State.ACTIVE, 'Active'),
25
+ (State.MAINTENANCE, 'Maintenance'),
26
+ )
@@ -0,0 +1,202 @@
1
+ import functools
2
+ import inspect
3
+ import types
4
+
5
+ from django.apps import AppConfig
6
+ from django.conf import settings
7
+ from django.contrib.auth import get_user_model
8
+ from django.core.exceptions import ImproperlyConfigured
9
+ from django.utils.module_loading import import_string
10
+
11
+ from djanquiltdb import ShardingMode
12
+ from djanquiltdb.db import connection
13
+ from djanquiltdb.decorators import class_method_use_shard, class_method_use_shard_from_db_arg
14
+ from djanquiltdb.options import ShardOptions
15
+ from djanquiltdb.postgresql_backend.base import ShardDatabaseWrapper
16
+ from djanquiltdb.utils import get_all_mirrored_models, get_all_public_models, get_all_sharded_models
17
+
18
+
19
+ class DjanQuiltDBConfig(AppConfig):
20
+ name = 'djanquiltdb'
21
+ verbose_name = 'DjanQuiltDB'
22
+
23
+ def ready(self):
24
+ from .models import BaseShard
25
+
26
+ if 'QUILT_DB' not in dir(settings) or not isinstance(settings.QUILT_DB, dict):
27
+ raise ImproperlyConfigured('Missing or incorrect type of a setting QUILT_DB.')
28
+
29
+ # Validate shard and node class settings
30
+ if 'SHARD_CLASS' not in settings.QUILT_DB:
31
+ raise ImproperlyConfigured('Missing or incorrect type of a setting QUILT_DB["{}"].'.format('SHARD_CLASS'))
32
+ class_ = import_string(settings.QUILT_DB['SHARD_CLASS'])
33
+ if not issubclass(class_, BaseShard):
34
+ raise ImproperlyConfigured(
35
+ 'The type {} should inherit from {}.'.format(settings.QUILT_DB['SHARD_CLASS'], BaseShard.__name__)
36
+ )
37
+ if hasattr(class_, '__sharding_mode') and getattr(class_, '__sharding_mode') == ShardingMode.SHARDED:
38
+ raise ImproperlyConfigured(
39
+ 'The Shard model cannot itself be sharded. It can only be non-sharded or mirrored.'
40
+ )
41
+
42
+ override_sharding_mode = settings.QUILT_DB.setdefault('OVERRIDE_SHARDING_MODE', {})
43
+ if not isinstance(override_sharding_mode, dict):
44
+ raise ImproperlyConfigured("Incorrect setting value of QUILT_DB['OVERRIDE_SHARDING_MODE'].")
45
+
46
+ for key, value in override_sharding_mode.items():
47
+ _validate_override_sharding_mode_entry(key, value)
48
+
49
+ # Convert app and model names to lowercase
50
+ settings.QUILT_DB['OVERRIDE_SHARDING_MODE'] = dict(
51
+ (tuple(x.lower() for x in k), v) for k, v in override_sharding_mode.items()
52
+ )
53
+
54
+ if (
55
+ 'DATABASE_ROUTERS' not in dir(settings)
56
+ or 'djanquiltdb.router.DynamicDbRouter' not in settings.DATABASE_ROUTERS
57
+ ):
58
+ raise ImproperlyConfigured(
59
+ 'djanquiltdb.router.DynamicDbRouter must be present in the DATABASE_ROUTERS setting.'
60
+ )
61
+
62
+ if (
63
+ 'SESSION_ENGINE' in dir(settings)
64
+ and settings.SESSION_ENGINE == 'django.contrib.sessions.backends.cached_db'
65
+ and getattr(get_user_model(), '__sharding_mode', False) in [ShardingMode.MIRRORED, ShardingMode.SHARDED]
66
+ ):
67
+ raise ImproperlyConfigured(
68
+ 'When the user model is sharded, you cannot use django.contrib.sessions.backends.cached_db '
69
+ "to store sessions. It references the user table and won't know where to find it."
70
+ )
71
+
72
+ if (
73
+ 'PRIMARY_DB_ALIAS' not in settings.QUILT_DB or not settings.QUILT_DB.get('PRIMARY_DB_ALIAS', None)
74
+ ) and get_all_mirrored_models():
75
+ raise ImproperlyConfigured("There are MIRRORED models, but QUILT_DB['PRIMARY_DB_ALIAS'] is not set.")
76
+
77
+ _validate_public_models()
78
+ _patch_connections()
79
+ _patch_transactions()
80
+ _initialize_sharded_models()
81
+
82
+
83
+ def _validate_public_models():
84
+ for model in get_all_public_models():
85
+ if not hasattr(model._meta, 'unique_together') or not model._meta.unique_together:
86
+ raise ImproperlyConfigured(
87
+ f'{model._meta.app_label}.{model.__name__} must have "unique_together" meta attribute set.'
88
+ )
89
+
90
+ if not hasattr(model, 'natural_key'):
91
+ raise ImproperlyConfigured(f'{model._meta.app_label}.{model.__name__} must define "natural_key" method.')
92
+
93
+ if not hasattr(model._default_manager, 'get_by_natural_key'):
94
+ raise ImproperlyConfigured(
95
+ f'The default manager for {model._meta.app_label}.{model.__name__} must'
96
+ 'define "get_by_natural_key" method.'
97
+ )
98
+
99
+
100
+ def _validate_override_sharding_mode_entry(key, value):
101
+ if not (isinstance(key, tuple) and len(key) in (1, 2) and isinstance(value, ShardingMode)):
102
+ raise ImproperlyConfigured(
103
+ 'The override sharding mode entry is improperly configured: {{ {}: {} }}'.format(repr(key), repr(value))
104
+ )
105
+
106
+
107
+ def _initialize_sharded_models():
108
+ """
109
+ Initialize sharded models by overriding all methods to add a use_shard context manager that makes sure all queries
110
+ are done in that same shard as the object is living in.
111
+ """
112
+ from_db_functions = {model: model.from_db for model in get_all_sharded_models(include_proxy=True)}
113
+
114
+ for model in get_all_sharded_models(include_proxy=True):
115
+ for attr, func in inspect.getmembers(model, inspect.isfunction):
116
+ # getattr(model, attr) will trigger dynamic lookup via the descriptor protocol, __getattr__ or
117
+ # __getattribute__. Therefore, we use inspect.getattr_static to strip out staticmethods (which we don't want
118
+ # to decorate).
119
+ if isinstance(inspect.getattr_static(model, attr), types.FunctionType):
120
+ # And decorate all model methods so that the methods will all run in the same shard context as the
121
+ # instance is living in
122
+ setattr(model, attr, class_method_use_shard(func))
123
+
124
+ # Setting the from_db function for a Model that is the parent of a ProxyModel will corrupt the ProxyModels
125
+ # version of the same function. So we have saved the original from_db function at the start, and use that.
126
+ model.add_to_class('from_db', class_method_use_shard_from_db_arg(from_db_functions[model]))
127
+
128
+ _initialize_sharded_model_querysets(model)
129
+
130
+
131
+ def post_init(func):
132
+ @functools.wraps(func)
133
+ def inner(self, *args, hints=None, **kwargs):
134
+ hints = hints or {}
135
+
136
+ if '_shard_options' not in hints and not connection.is_public_schema():
137
+ hints['_shard_options'] = connection.shard_options
138
+
139
+ func(self, *args, hints=hints, **kwargs)
140
+
141
+ inner.__decorator__ = post_init
142
+ return inner
143
+
144
+
145
+ def _initialize_sharded_model_querysets(model):
146
+ """
147
+ Override all the querysets of the managers, so they can remember the shard where they are initialized on
148
+ """
149
+ for instance in model._meta.managers:
150
+ if (
151
+ hasattr(instance._queryset_class.__init__, '__decorator__')
152
+ and instance._queryset_class.__init__.__decorator__ == post_init
153
+ ):
154
+ continue
155
+ setattr(instance._queryset_class, '__init__', post_init(instance._queryset_class.__init__))
156
+
157
+
158
+ def patch_getitem(func):
159
+ @functools.wraps(func)
160
+ def inner(self, alias):
161
+ # If we're planning to just go into the public schema, then we're going to use the normal connection for that
162
+ if isinstance(alias, str) and '|' not in alias or isinstance(alias, ShardOptions) and alias.is_public_schema():
163
+ return func(self, alias if isinstance(alias, str) else alias.node_name)
164
+
165
+ options = ShardOptions.from_alias(alias)
166
+
167
+ # Retrieves the main connection to the database, so we can still do connection pooling
168
+ connection_ = func(self, options.node_name)
169
+
170
+ # Sets up the sharded connection
171
+ return ShardDatabaseWrapper(connection_, options)
172
+
173
+ return inner
174
+
175
+
176
+ def _patch_connections():
177
+ """
178
+ Monkeypatch django.db.connection and django.db.connections to accept:
179
+
180
+ * node_name
181
+ * node_name|schema_name
182
+ * tuple (node_name, schema_name)
183
+ * ShardOptions instance
184
+ * Shard model instance
185
+ """
186
+ import django.db
187
+ from django.db.utils import ConnectionHandler
188
+
189
+ setattr(ConnectionHandler, '__getitem__', patch_getitem(ConnectionHandler.__getitem__))
190
+ setattr(django.db, 'connection', connection)
191
+
192
+
193
+ def _patch_transactions():
194
+ """
195
+ Monkeypatch django.db.transaction.get_connection to use the active node, not always the default:
196
+ """
197
+ import django.db.transaction
198
+
199
+ from djanquiltdb.transaction import atomic, get_connection
200
+
201
+ setattr(django.db.transaction, 'get_connection', get_connection)
202
+ setattr(django.db.transaction, 'atomic', atomic)
@@ -0,0 +1,103 @@
1
+ import progressbar
2
+ from django.db.models.deletion import Collector
3
+ from django.db.models.deletion import get_candidate_relations_to_delete as get_candidate_relations_to_collect
4
+
5
+
6
+ class SimpleCollector(Collector):
7
+ """
8
+ Simple collector does basically the same as the default Django collector, but also follows the on delete SET NULL
9
+ relations. We need this to move data from one shard to another.
10
+ """
11
+
12
+ def __init__(self, connection, verbose=False):
13
+ super().__init__(using=connection.alias)
14
+
15
+ self.connection = connection
16
+ self.verbose = verbose
17
+ self.data_points = 0
18
+
19
+ if self.verbose:
20
+ self.bar = progressbar.ProgressBar(
21
+ max_value=progressbar.UnknownLength,
22
+ widgets=[
23
+ progressbar.RotatingMarker(),
24
+ ' Collected ',
25
+ progressbar.Counter(),
26
+ ' datapoints; ',
27
+ progressbar.Timer(),
28
+ ],
29
+ )
30
+
31
+ def add(self, *args, **kwargs):
32
+ new_objs = super().add(*args, **kwargs)
33
+
34
+ if self.verbose:
35
+ self.data_points += len(new_objs)
36
+ self.bar.update(self.data_points)
37
+
38
+ return new_objs
39
+
40
+ def get_batches(self, objs, field):
41
+ """
42
+ Returns the objs in suitably sized batches for the used connection.
43
+ """
44
+ conn_batch_size = max(self.connection.ops.bulk_batch_size([field.name], objs), 1)
45
+ if len(objs) > conn_batch_size:
46
+ return [objs[i : i + conn_batch_size] for i in range(0, len(objs), conn_batch_size)]
47
+ else:
48
+ return [objs]
49
+
50
+ def finish_bar(self):
51
+ if self.verbose:
52
+ self.bar.finish()
53
+
54
+ def collect(
55
+ self,
56
+ objs,
57
+ source=None,
58
+ nullable=False,
59
+ collect_related=True,
60
+ source_attr=None,
61
+ reverse_dependency=False,
62
+ keep_parents=False,
63
+ fail_on_restricted=True,
64
+ ):
65
+ """
66
+ Kind of the same as the original collector, but now also follows the on delete SET NULL relations.
67
+ """
68
+ new_objs = self.add(objs, source, nullable, reverse_dependency=reverse_dependency)
69
+
70
+ if not new_objs:
71
+ self.finish_bar()
72
+ return
73
+
74
+ model = new_objs[0].__class__
75
+
76
+ # Recursively collect concrete model's parent models, but not their
77
+ # related objects. These will be found by meta.get_fields()
78
+ concrete_model = model._meta.concrete_model
79
+ for ptr in concrete_model._meta.parents.values():
80
+ if ptr:
81
+ # FIXME: This seems to be buggy and execute a query for each
82
+ # parent object fetch. We have the parent data in the obj,
83
+ # but we don't have a nice way to turn that data into parent
84
+ # object instance.
85
+ parent_objs = [getattr(obj, ptr.name) for obj in new_objs]
86
+ self.collect(
87
+ parent_objs, source=model, source_attr=ptr.remote_field.related_name, collect_related=False
88
+ )
89
+
90
+ if collect_related:
91
+ for related in get_candidate_relations_to_collect(model._meta):
92
+ field = related.field
93
+ batches = self.get_batches(new_objs, field)
94
+ for batch in batches:
95
+ sub_objs = self.related_objects(related.related_model, [field], batch)
96
+ if not sub_objs:
97
+ continue
98
+ self.collect(sub_objs, source=model, source_attr=field.name)
99
+ for field in model._meta.private_fields:
100
+ if hasattr(field, 'bulk_related_objects'):
101
+ # It's something like generic foreign key.
102
+ sub_objs = field.bulk_related_objects(new_objs, self.using)
103
+ self.collect(sub_objs, source=model, nullable=True, fail_on_restricted=False)
@@ -0,0 +1,34 @@
1
+ from typing import TYPE_CHECKING, Optional
2
+
3
+ from django.apps import AppConfig
4
+ from django.core.exceptions import ImproperlyConfigured
5
+
6
+ if TYPE_CHECKING:
7
+ from djanquiltdb.contrib.quilt_admin.shard_selector import BaseAdminShardSelector
8
+
9
+ ADMIN_SHARD_SELECTOR_CLASS: Optional[BaseAdminShardSelector] = None
10
+
11
+
12
+ class ShardedAdminAppConfig(AppConfig):
13
+ name = 'djanquiltdb.contrib.quilt_admin'
14
+ verbose_name = 'DjanQuiltDB Admin Integration'
15
+
16
+ def ready(self):
17
+ from django.conf import settings
18
+ from django.utils.module_loading import import_string
19
+
20
+ from djanquiltdb.contrib.quilt_admin.shard_selector import BaseAdminShardSelector
21
+
22
+ global ADMIN_SHARD_SELECTOR_CLASS
23
+
24
+ if hasattr(settings, 'QUILT_ADMIN') and 'SHARD_SELECTOR_CLASS' in settings.QUILT_ADMIN:
25
+ shard_selector_class = settings.QUILT_ADMIN['SHARD_SELECTOR_CLASS']
26
+ else:
27
+ shard_selector_class = 'djanquiltdb.contrib.quilt_admin.shard_selector.AdminShardSelector'
28
+
29
+ ADMIN_SHARD_SELECTOR_CLASS = import_string(shard_selector_class)
30
+
31
+ if not issubclass(ADMIN_SHARD_SELECTOR_CLASS, BaseAdminShardSelector):
32
+ raise ImproperlyConfigured(
33
+ f"Shard selector class {ADMIN_SHARD_SELECTOR_CLASS} doesn't subclass BaseAdminShardSelector"
34
+ )
@@ -0,0 +1,85 @@
1
+ from djanquiltdb.contrib.quilt_admin.apps import ADMIN_SHARD_SELECTOR_CLASS
2
+ from djanquiltdb.utils import get_mapping_class, get_shard_class
3
+
4
+
5
+ def admin_shard_context(request):
6
+ """
7
+ Context processor to provide shard information to admin templates.
8
+ Adds shard switcher context to admin templates.
9
+
10
+ If a mapping model is configured (QUILT_DB["MAPPING_MODEL"]), the selector lists
11
+ mapping entries (by mapping value) instead of shards (by shard id).
12
+ """
13
+ context = {
14
+ 'available_shards': [],
15
+ 'current_shard_id': None, # kept for backwards compatibility
16
+ 'shard_switcher_options': [],
17
+ 'shard_switcher_field_name': 'shard_id',
18
+ 'current_shard_switcher_value': '',
19
+ 'shard_switcher_mode': 'shard', # or 'mapping'
20
+ 'shard_maintenance_mode': False,
21
+ 'shard_maintenance_message': None,
22
+ }
23
+ # Add maintenance status if available
24
+ if hasattr(request, '_shard_maintenance_mode'):
25
+ context['shard_maintenance_mode'] = request._shard_maintenance_mode
26
+ context['shard_maintenance_message'] = getattr(request, '_shard_maintenance_message', None)
27
+ # Only add shard context if user is authenticated and in admin
28
+ if hasattr(request, 'user') and request.user.is_authenticated and request.path.startswith('/admin/'):
29
+ try:
30
+ mapping_class = get_mapping_class()
31
+ if mapping_class:
32
+ mapping_field = getattr(mapping_class, 'mapping_field', None)
33
+
34
+ qs = mapping_class.objects.using('default').select_related('shard').all()
35
+ if mapping_field:
36
+ qs = qs.order_by(mapping_field)
37
+ else:
38
+ qs = qs.order_by('pk')
39
+
40
+ mappings = list(qs)
41
+ context['available_shards'] = mappings
42
+ context['shard_switcher_mode'] = 'mapping'
43
+ context['shard_switcher_field_name'] = 'mapping_value'
44
+
45
+ current_value = ADMIN_SHARD_SELECTOR_CLASS.retrieve_override_value(request)
46
+ context['current_shard_switcher_value'] = '' if current_value is None else str(current_value)
47
+
48
+ options = []
49
+ for mapping in mappings:
50
+ if mapping_field:
51
+ mapping_value = getattr(mapping, mapping_field)
52
+ mapping_value_str = '' if mapping_value is None else str(mapping_value)
53
+ else:
54
+ mapping_value = mapping.pk
55
+ mapping_value_str = str(mapping.pk)
56
+
57
+ shard = getattr(mapping, 'shard', None)
58
+ label = ADMIN_SHARD_SELECTOR_CLASS.format_override_option(mapping_value, shard, mapping)
59
+
60
+ options.append({'value': mapping_value_str, 'label': label})
61
+
62
+ context['shard_switcher_options'] = options
63
+ else:
64
+ shard_class = get_shard_class()
65
+ # Get all shards from the primary database
66
+ # Using .using('default') to ensure we query from the primary database
67
+ shards = list(shard_class.objects.using('default').all().order_by('alias'))
68
+ context['available_shards'] = shards
69
+
70
+ current_id = ADMIN_SHARD_SELECTOR_CLASS.retrieve_override_value(request)
71
+ context['current_shard_id'] = current_id
72
+ context['current_shard_switcher_value'] = '' if current_id is None else str(current_id)
73
+
74
+ context['shard_switcher_options'] = [
75
+ {
76
+ 'value': str(shard.id),
77
+ 'label': ADMIN_SHARD_SELECTOR_CLASS.format_override_option(shard.id, shard),
78
+ }
79
+ for shard in shards
80
+ ]
81
+ except Exception:
82
+ # If there's any error (e.g., database not ready, no shards exist), just return empty context
83
+ pass
84
+
85
+ return context