django-concurrent-test 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- django_concurrent_test/__init__.py +23 -0
- django_concurrent_test/db.py +1309 -0
- django_concurrent_test/exceptions.py +43 -0
- django_concurrent_test/middleware.py +487 -0
- django_concurrent_test/pytest_plugin.py +319 -0
- django_concurrent_test/runner.py +1325 -0
- django_concurrent_test/security.py +321 -0
- django_concurrent_test/timing_utils.py +316 -0
- django_concurrent_test-1.0.0.dist-info/METADATA +554 -0
- django_concurrent_test-1.0.0.dist-info/RECORD +15 -0
- django_concurrent_test-1.0.0.dist-info/WHEEL +5 -0
- django_concurrent_test-1.0.0.dist-info/entry_points.txt +2 -0
- django_concurrent_test-1.0.0.dist-info/licenses/AUTHORS.md +35 -0
- django_concurrent_test-1.0.0.dist-info/licenses/LICENSE +21 -0
- django_concurrent_test-1.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1309 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Database utilities for django-concurrent-test package.
|
|
3
|
+
|
|
4
|
+
This module provides thread-safe database cloning with template caching,
|
|
5
|
+
connection pooling, and batch operations for optimal performance in
|
|
6
|
+
concurrent testing scenarios.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import os
|
|
10
|
+
import time
|
|
11
|
+
import logging
|
|
12
|
+
import threading
|
|
13
|
+
from contextlib import contextmanager
|
|
14
|
+
from typing import List, Dict, Optional, Set
|
|
15
|
+
from django.db import connections, connection
|
|
16
|
+
from django.conf import settings
|
|
17
|
+
|
|
18
|
+
from .exceptions import (
|
|
19
|
+
DatabaseCloneException,
|
|
20
|
+
UnsupportedDatabase,
|
|
21
|
+
PermissionException,
|
|
22
|
+
)
|
|
23
|
+
from .security import (
|
|
24
|
+
get_safe_worker_database_name,
|
|
25
|
+
validate_database_permissions,
|
|
26
|
+
sanitize_database_name,
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
logger = logging.getLogger(__name__)
|
|
30
|
+
|
|
31
|
+
# Thread-safe template cache
|
|
32
|
+
_template_cache = {}
|
|
33
|
+
_template_cache_lock = threading.Lock()
|
|
34
|
+
|
|
35
|
+
# Template fingerprinting for cache invalidation
|
|
36
|
+
_template_fingerprints = {}
|
|
37
|
+
_template_fingerprint_lock = threading.Lock()
|
|
38
|
+
|
|
39
|
+
# Connection pool for worker databases
|
|
40
|
+
_connection_pool = {}
|
|
41
|
+
_connection_pool_lock = threading.Lock()
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class DatabaseCloner:
|
|
45
|
+
"""Base class for database cloning operations."""
|
|
46
|
+
|
|
47
|
+
def __init__(self, connection):
|
|
48
|
+
self.connection = connection
|
|
49
|
+
self.vendor = connection.vendor
|
|
50
|
+
|
|
51
|
+
def clone_database(self, worker_id):
|
|
52
|
+
"""
|
|
53
|
+
Clone database for worker.
|
|
54
|
+
|
|
55
|
+
Args:
|
|
56
|
+
worker_id (int): Worker ID
|
|
57
|
+
|
|
58
|
+
Returns:
|
|
59
|
+
str: New database name
|
|
60
|
+
|
|
61
|
+
Raises:
|
|
62
|
+
DatabaseCloneException: If cloning fails
|
|
63
|
+
"""
|
|
64
|
+
raise NotImplementedError
|
|
65
|
+
|
|
66
|
+
def drop_database(self, db_name):
|
|
67
|
+
"""
|
|
68
|
+
Drop database.
|
|
69
|
+
|
|
70
|
+
Args:
|
|
71
|
+
db_name (str): Database name to drop
|
|
72
|
+
|
|
73
|
+
Raises:
|
|
74
|
+
DatabaseCloneException: If dropping fails
|
|
75
|
+
"""
|
|
76
|
+
raise NotImplementedError
|
|
77
|
+
|
|
78
|
+
def database_exists(self, db_name):
|
|
79
|
+
"""
|
|
80
|
+
Check if database exists.
|
|
81
|
+
|
|
82
|
+
Args:
|
|
83
|
+
db_name (str): Database name to check
|
|
84
|
+
|
|
85
|
+
Returns:
|
|
86
|
+
bool: True if database exists
|
|
87
|
+
"""
|
|
88
|
+
raise NotImplementedError
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class PostgreSQLCloner(DatabaseCloner):
|
|
92
|
+
"""
|
|
93
|
+
PostgreSQL database cloner using template0 for optimal performance.
|
|
94
|
+
|
|
95
|
+
This cloner implements template caching and batch operations for efficient
|
|
96
|
+
database cloning in concurrent testing scenarios. It uses TEMPLATE template0
|
|
97
|
+
to ensure clean schema cloning without data.
|
|
98
|
+
"""
|
|
99
|
+
|
|
100
|
+
def __init__(self, connection):
|
|
101
|
+
super().__init__(connection)
|
|
102
|
+
self._template_db_name = None
|
|
103
|
+
self._template_created = False
|
|
104
|
+
|
|
105
|
+
def _ensure_template_database(self):
|
|
106
|
+
"""
|
|
107
|
+
Ensure template database exists and is properly configured.
|
|
108
|
+
|
|
109
|
+
Creates a template database from the base database if it doesn't exist,
|
|
110
|
+
ensuring it has the correct schema without any data. This template is
|
|
111
|
+
cached and reused for all subsequent worker database creation.
|
|
112
|
+
|
|
113
|
+
Thread-safe implementation using locks to prevent race conditions
|
|
114
|
+
during template creation. Includes template versioning for cache invalidation.
|
|
115
|
+
"""
|
|
116
|
+
global _template_cache, _template_cache_lock, _template_fingerprints, _template_fingerprint_lock
|
|
117
|
+
|
|
118
|
+
db_config = self.connection.settings_dict
|
|
119
|
+
base_name = db_config['NAME']
|
|
120
|
+
db_user = db_config.get('USER', 'postgres')
|
|
121
|
+
template_key = f"{base_name}_{db_user}"
|
|
122
|
+
|
|
123
|
+
# Generate current schema fingerprint
|
|
124
|
+
current_fingerprint = generate_template_fingerprint(self.connection)
|
|
125
|
+
|
|
126
|
+
with _template_cache_lock:
|
|
127
|
+
# Check if template exists and version matches
|
|
128
|
+
if template_key in _template_cache:
|
|
129
|
+
cached_fingerprint = _template_fingerprints.get(template_key)
|
|
130
|
+
|
|
131
|
+
# If fingerprint matches, use cached template
|
|
132
|
+
if cached_fingerprint == current_fingerprint:
|
|
133
|
+
self._template_db_name = _template_cache[template_key]
|
|
134
|
+
logger.debug(f"Using cached template {self._template_db_name} (fingerprint: {current_fingerprint[:8]})")
|
|
135
|
+
return
|
|
136
|
+
else:
|
|
137
|
+
# Template version mismatch, refresh template
|
|
138
|
+
logger.info(f"Template version mismatch, refreshing template (old: {cached_fingerprint[:8]}, new: {current_fingerprint[:8]})")
|
|
139
|
+
self._refresh_template(template_key)
|
|
140
|
+
|
|
141
|
+
# Create template database name
|
|
142
|
+
template_db_name = f"{base_name}_template"
|
|
143
|
+
|
|
144
|
+
try:
|
|
145
|
+
with self.connection.cursor() as cursor:
|
|
146
|
+
# Check if template already exists
|
|
147
|
+
if not self.database_exists(template_db_name):
|
|
148
|
+
logger.info(f"Creating template database: {template_db_name}")
|
|
149
|
+
|
|
150
|
+
# Create template from base database
|
|
151
|
+
cursor.execute(
|
|
152
|
+
f"CREATE DATABASE {template_db_name} "
|
|
153
|
+
f"TEMPLATE {base_name} WITH OWNER {db_user}"
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
# Ensure template is marked as template
|
|
157
|
+
cursor.execute(
|
|
158
|
+
f"UPDATE pg_database SET datistemplate = true "
|
|
159
|
+
f"WHERE datname = '{template_db_name}'"
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
logger.info(f"Template database {template_db_name} created successfully")
|
|
163
|
+
else:
|
|
164
|
+
logger.info(f"Using existing template database: {template_db_name}")
|
|
165
|
+
|
|
166
|
+
# Cache the template database name and fingerprint
|
|
167
|
+
_template_cache[template_key] = template_db_name
|
|
168
|
+
_template_fingerprints[template_key] = current_fingerprint
|
|
169
|
+
self._template_db_name = template_db_name
|
|
170
|
+
|
|
171
|
+
except Exception as e:
|
|
172
|
+
raise DatabaseCloneException(
|
|
173
|
+
f"Failed to create template database: {e}"
|
|
174
|
+
) from e
|
|
175
|
+
|
|
176
|
+
def _refresh_template(self, template_key: str):
|
|
177
|
+
"""
|
|
178
|
+
Refresh template database when schema changes.
|
|
179
|
+
|
|
180
|
+
Args:
|
|
181
|
+
template_key (str): Template cache key
|
|
182
|
+
"""
|
|
183
|
+
global _template_cache, _template_fingerprints
|
|
184
|
+
|
|
185
|
+
try:
|
|
186
|
+
# Drop existing template
|
|
187
|
+
if self._template_db_name and self.database_exists(self._template_db_name):
|
|
188
|
+
self.drop_database(self._template_db_name)
|
|
189
|
+
logger.info(f"Dropped outdated template: {self._template_db_name}")
|
|
190
|
+
|
|
191
|
+
# Clear cache entries
|
|
192
|
+
if template_key in _template_cache:
|
|
193
|
+
del _template_cache[template_key]
|
|
194
|
+
if template_key in _template_fingerprints:
|
|
195
|
+
del _template_fingerprints[template_key]
|
|
196
|
+
|
|
197
|
+
# Reset template state
|
|
198
|
+
self._template_db_name = None
|
|
199
|
+
self._template_created = False
|
|
200
|
+
|
|
201
|
+
logger.info("Template cache cleared, will recreate on next use")
|
|
202
|
+
|
|
203
|
+
except Exception as e:
|
|
204
|
+
logger.warning(f"Failed to refresh template: {e}")
|
|
205
|
+
# Continue with template creation
|
|
206
|
+
|
|
207
|
+
def clone_database(self, worker_id):
|
|
208
|
+
"""
|
|
209
|
+
Clone PostgreSQL database using template0 for optimal performance.
|
|
210
|
+
|
|
211
|
+
Uses a cached template database to create worker databases efficiently.
|
|
212
|
+
The template is created once and reused for all subsequent workers,
|
|
213
|
+
ensuring consistent schema and optimal performance.
|
|
214
|
+
|
|
215
|
+
Args:
|
|
216
|
+
worker_id (int): Worker ID
|
|
217
|
+
|
|
218
|
+
Returns:
|
|
219
|
+
str: New database name
|
|
220
|
+
|
|
221
|
+
Raises:
|
|
222
|
+
DatabaseCloneException: If cloning fails
|
|
223
|
+
"""
|
|
224
|
+
# Ensure template database exists
|
|
225
|
+
self._ensure_template_database()
|
|
226
|
+
|
|
227
|
+
# Generate safe worker database name
|
|
228
|
+
worker_db_name = get_safe_worker_database_name(
|
|
229
|
+
self.connection.settings_dict['NAME'],
|
|
230
|
+
worker_id
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
try:
|
|
234
|
+
with self.connection.cursor() as cursor:
|
|
235
|
+
# Check if worker database already exists
|
|
236
|
+
if self.database_exists(worker_db_name):
|
|
237
|
+
logger.warning(f"Database {worker_db_name} already exists, dropping it")
|
|
238
|
+
self.drop_database(worker_db_name)
|
|
239
|
+
|
|
240
|
+
# Create database from template0 (clean template)
|
|
241
|
+
cursor.execute(
|
|
242
|
+
f"CREATE DATABASE {worker_db_name} "
|
|
243
|
+
f"TEMPLATE template0"
|
|
244
|
+
)
|
|
245
|
+
|
|
246
|
+
# Apply schema from our template database
|
|
247
|
+
cursor.execute(
|
|
248
|
+
f"SELECT pg_restore_schema('{self._template_db_name}', '{worker_db_name}')"
|
|
249
|
+
)
|
|
250
|
+
|
|
251
|
+
logger.info(f"Created PostgreSQL database: {worker_db_name}")
|
|
252
|
+
return worker_db_name
|
|
253
|
+
|
|
254
|
+
except Exception as e:
|
|
255
|
+
raise DatabaseCloneException(
|
|
256
|
+
f"Failed to clone PostgreSQL database: {e}"
|
|
257
|
+
) from e
|
|
258
|
+
|
|
259
|
+
def clone_databases_batch(self, worker_ids: List[int]) -> List[str]:
|
|
260
|
+
"""
|
|
261
|
+
Clone multiple databases in a single transaction for optimal performance.
|
|
262
|
+
|
|
263
|
+
This method creates multiple worker databases in a single transaction,
|
|
264
|
+
reducing overhead and improving performance for concurrent testing.
|
|
265
|
+
|
|
266
|
+
Args:
|
|
267
|
+
worker_ids (List[int]): List of worker IDs
|
|
268
|
+
|
|
269
|
+
Returns:
|
|
270
|
+
List[str]: List of created database names
|
|
271
|
+
|
|
272
|
+
Raises:
|
|
273
|
+
DatabaseCloneException: If batch cloning fails
|
|
274
|
+
"""
|
|
275
|
+
# Ensure template database exists
|
|
276
|
+
self._ensure_template_database()
|
|
277
|
+
|
|
278
|
+
# Generate all database names
|
|
279
|
+
db_names = [
|
|
280
|
+
get_safe_worker_database_name(
|
|
281
|
+
self.connection.settings_dict['NAME'],
|
|
282
|
+
worker_id
|
|
283
|
+
)
|
|
284
|
+
for worker_id in worker_ids
|
|
285
|
+
]
|
|
286
|
+
|
|
287
|
+
try:
|
|
288
|
+
with self.connection.cursor() as cursor:
|
|
289
|
+
# Start transaction
|
|
290
|
+
cursor.execute("BEGIN")
|
|
291
|
+
|
|
292
|
+
try:
|
|
293
|
+
for db_name in db_names:
|
|
294
|
+
# Check if database already exists
|
|
295
|
+
if self.database_exists(db_name):
|
|
296
|
+
logger.warning(f"Database {db_name} already exists, dropping it")
|
|
297
|
+
self.drop_database(db_name)
|
|
298
|
+
|
|
299
|
+
# Create database from template0
|
|
300
|
+
cursor.execute(
|
|
301
|
+
f"CREATE DATABASE {db_name} "
|
|
302
|
+
f"TEMPLATE template0"
|
|
303
|
+
)
|
|
304
|
+
|
|
305
|
+
# Apply schema from template
|
|
306
|
+
cursor.execute(
|
|
307
|
+
f"SELECT pg_restore_schema('{self._template_db_name}', '{db_name}')"
|
|
308
|
+
)
|
|
309
|
+
|
|
310
|
+
# Commit transaction
|
|
311
|
+
cursor.execute("COMMIT")
|
|
312
|
+
|
|
313
|
+
logger.info(f"Created {len(db_names)} PostgreSQL databases in batch")
|
|
314
|
+
return db_names
|
|
315
|
+
|
|
316
|
+
except Exception as e:
|
|
317
|
+
# Rollback on error
|
|
318
|
+
cursor.execute("ROLLBACK")
|
|
319
|
+
raise e
|
|
320
|
+
|
|
321
|
+
except Exception as e:
|
|
322
|
+
raise DatabaseCloneException(
|
|
323
|
+
f"Failed to clone PostgreSQL databases in batch: {e}"
|
|
324
|
+
) from e
|
|
325
|
+
|
|
326
|
+
def drop_database(self, db_name):
|
|
327
|
+
"""
|
|
328
|
+
Drop PostgreSQL database.
|
|
329
|
+
|
|
330
|
+
Args:
|
|
331
|
+
db_name (str): Database name to drop
|
|
332
|
+
|
|
333
|
+
Raises:
|
|
334
|
+
DatabaseCloneException: If dropping fails
|
|
335
|
+
"""
|
|
336
|
+
from django.db import connection
|
|
337
|
+
|
|
338
|
+
try:
|
|
339
|
+
with connection.cursor() as cursor:
|
|
340
|
+
# Terminate connections to the database
|
|
341
|
+
cursor.execute(
|
|
342
|
+
f"SELECT pg_terminate_backend(pid) "
|
|
343
|
+
f"FROM pg_stat_activity "
|
|
344
|
+
f"WHERE datname = '{db_name}' AND pid <> pg_backend_pid()"
|
|
345
|
+
)
|
|
346
|
+
|
|
347
|
+
# Drop the database
|
|
348
|
+
cursor.execute(f"DROP DATABASE IF EXISTS {db_name}")
|
|
349
|
+
logger.info(f"Dropped PostgreSQL database: {db_name}")
|
|
350
|
+
|
|
351
|
+
except Exception as e:
|
|
352
|
+
raise DatabaseCloneException(
|
|
353
|
+
f"Failed to drop PostgreSQL database {db_name}: {e}"
|
|
354
|
+
) from e
|
|
355
|
+
|
|
356
|
+
def database_exists(self, db_name):
|
|
357
|
+
"""
|
|
358
|
+
Check if PostgreSQL database exists.
|
|
359
|
+
|
|
360
|
+
Args:
|
|
361
|
+
db_name (str): Database name to check
|
|
362
|
+
|
|
363
|
+
Returns:
|
|
364
|
+
bool: True if database exists
|
|
365
|
+
"""
|
|
366
|
+
from django.db import connection
|
|
367
|
+
|
|
368
|
+
try:
|
|
369
|
+
with connection.cursor() as cursor:
|
|
370
|
+
cursor.execute(
|
|
371
|
+
"SELECT 1 FROM pg_database WHERE datname = %s",
|
|
372
|
+
[db_name]
|
|
373
|
+
)
|
|
374
|
+
return cursor.fetchone() is not None
|
|
375
|
+
except Exception:
|
|
376
|
+
return False
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
class SQLiteCloner(DatabaseCloner):
|
|
380
|
+
"""
|
|
381
|
+
SQLite database cloner that skips cloning but marks tests as sequential.
|
|
382
|
+
|
|
383
|
+
This cloner is designed for local development where SQLite is used.
|
|
384
|
+
Since SQLite doesn't support database cloning, it marks tests as sequential
|
|
385
|
+
to ensure proper isolation.
|
|
386
|
+
"""
|
|
387
|
+
|
|
388
|
+
def __init__(self, connection):
|
|
389
|
+
super().__init__(connection)
|
|
390
|
+
logger.warning("SQLite detected - tests will run sequentially for proper isolation")
|
|
391
|
+
|
|
392
|
+
def clone_database(self, worker_id):
|
|
393
|
+
"""
|
|
394
|
+
SQLite doesn't support database cloning - return original database name.
|
|
395
|
+
|
|
396
|
+
Args:
|
|
397
|
+
worker_id (int): Worker ID (ignored for SQLite)
|
|
398
|
+
|
|
399
|
+
Returns:
|
|
400
|
+
str: Original database name
|
|
401
|
+
|
|
402
|
+
Raises:
|
|
403
|
+
DatabaseCloneException: Always raised for SQLite
|
|
404
|
+
"""
|
|
405
|
+
raise DatabaseCloneException(
|
|
406
|
+
"SQLite doesn't support database cloning. "
|
|
407
|
+
"Tests will run sequentially for proper isolation."
|
|
408
|
+
)
|
|
409
|
+
|
|
410
|
+
def clone_databases_batch(self, worker_ids: List[int]) -> List[str]:
|
|
411
|
+
"""
|
|
412
|
+
SQLite doesn't support batch database cloning.
|
|
413
|
+
|
|
414
|
+
Args:
|
|
415
|
+
worker_ids (List[int]): List of worker IDs (ignored for SQLite)
|
|
416
|
+
|
|
417
|
+
Returns:
|
|
418
|
+
List[str]: Empty list since cloning is not supported
|
|
419
|
+
|
|
420
|
+
Raises:
|
|
421
|
+
DatabaseCloneException: Always raised for SQLite
|
|
422
|
+
"""
|
|
423
|
+
raise DatabaseCloneException(
|
|
424
|
+
"SQLite doesn't support batch database cloning. "
|
|
425
|
+
"Tests will run sequentially for proper isolation."
|
|
426
|
+
)
|
|
427
|
+
|
|
428
|
+
def drop_database(self, db_name):
|
|
429
|
+
"""
|
|
430
|
+
SQLite doesn't support dropping databases.
|
|
431
|
+
|
|
432
|
+
Args:
|
|
433
|
+
db_name (str): Database name (ignored for SQLite)
|
|
434
|
+
"""
|
|
435
|
+
logger.info(f"SQLite: Skipping database drop for {db_name}")
|
|
436
|
+
|
|
437
|
+
def database_exists(self, db_name):
|
|
438
|
+
"""
|
|
439
|
+
Check if SQLite database exists.
|
|
440
|
+
|
|
441
|
+
Args:
|
|
442
|
+
db_name (str): Database name to check
|
|
443
|
+
|
|
444
|
+
Returns:
|
|
445
|
+
bool: True if database file exists
|
|
446
|
+
"""
|
|
447
|
+
import os
|
|
448
|
+
return os.path.exists(db_name)
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
class MySQLCloner(DatabaseCloner):
|
|
452
|
+
"""
|
|
453
|
+
MySQL database cloner using schema replication with IGNORE DATA clause.
|
|
454
|
+
|
|
455
|
+
This cloner implements template caching and batch operations for efficient
|
|
456
|
+
database cloning in concurrent testing scenarios. It uses IGNORE DATA to
|
|
457
|
+
ensure clean schema cloning without data.
|
|
458
|
+
"""
|
|
459
|
+
|
|
460
|
+
def __init__(self, connection):
|
|
461
|
+
super().__init__(connection)
|
|
462
|
+
self._template_db_name = None
|
|
463
|
+
self._template_created = False
|
|
464
|
+
|
|
465
|
+
def _ensure_template_database(self):
|
|
466
|
+
"""
|
|
467
|
+
Ensure template database exists and is properly configured.
|
|
468
|
+
|
|
469
|
+
Creates a template database from the base database if it doesn't exist,
|
|
470
|
+
ensuring it has the correct schema without any data. This template is
|
|
471
|
+
cached and reused for all subsequent worker database creation.
|
|
472
|
+
|
|
473
|
+
Thread-safe implementation using locks to prevent race conditions
|
|
474
|
+
during template creation.
|
|
475
|
+
"""
|
|
476
|
+
global _template_cache, _template_cache_lock
|
|
477
|
+
|
|
478
|
+
db_config = self.connection.settings_dict
|
|
479
|
+
base_name = db_config['NAME']
|
|
480
|
+
template_key = f"{base_name}_mysql"
|
|
481
|
+
|
|
482
|
+
with _template_cache_lock:
|
|
483
|
+
if template_key in _template_cache:
|
|
484
|
+
self._template_db_name = _template_cache[template_key]
|
|
485
|
+
return
|
|
486
|
+
|
|
487
|
+
# Create template database name
|
|
488
|
+
template_db_name = f"{base_name}_template"
|
|
489
|
+
|
|
490
|
+
try:
|
|
491
|
+
with self.connection.cursor() as cursor:
|
|
492
|
+
# Check if template already exists
|
|
493
|
+
if not self.database_exists(template_db_name):
|
|
494
|
+
logger.info(f"Creating MySQL template database: {template_db_name}")
|
|
495
|
+
|
|
496
|
+
# Create template database
|
|
497
|
+
cursor.execute(f"CREATE DATABASE {template_db_name}")
|
|
498
|
+
|
|
499
|
+
# Get all tables from base database
|
|
500
|
+
cursor.execute(
|
|
501
|
+
"SELECT table_name FROM information_schema.tables "
|
|
502
|
+
"WHERE table_schema = %s",
|
|
503
|
+
[base_name]
|
|
504
|
+
)
|
|
505
|
+
tables = [row[0] for row in cursor.fetchall()]
|
|
506
|
+
|
|
507
|
+
# Clone schema for each table with IGNORE DATA
|
|
508
|
+
for table in tables:
|
|
509
|
+
cursor.execute(
|
|
510
|
+
f"CREATE TABLE {template_db_name}.{table} "
|
|
511
|
+
f"LIKE {base_name}.{table} IGNORE DATA"
|
|
512
|
+
)
|
|
513
|
+
|
|
514
|
+
logger.info(f"MySQL template database {template_db_name} created successfully")
|
|
515
|
+
else:
|
|
516
|
+
logger.info(f"Using existing MySQL template database: {template_db_name}")
|
|
517
|
+
|
|
518
|
+
# Cache the template database name
|
|
519
|
+
_template_cache[template_key] = template_db_name
|
|
520
|
+
self._template_db_name = template_db_name
|
|
521
|
+
|
|
522
|
+
except Exception as e:
|
|
523
|
+
raise DatabaseCloneException(
|
|
524
|
+
f"Failed to create MySQL template database: {e}"
|
|
525
|
+
) from e
|
|
526
|
+
|
|
527
|
+
def clone_database(self, worker_id):
|
|
528
|
+
"""
|
|
529
|
+
Clone MySQL database using schema replication with IGNORE DATA clause.
|
|
530
|
+
|
|
531
|
+
Uses a cached template database to create worker databases efficiently.
|
|
532
|
+
The template is created once and reused for all subsequent workers,
|
|
533
|
+
ensuring consistent schema and optimal performance.
|
|
534
|
+
|
|
535
|
+
Args:
|
|
536
|
+
worker_id (int): Worker ID
|
|
537
|
+
|
|
538
|
+
Returns:
|
|
539
|
+
str: New database name
|
|
540
|
+
|
|
541
|
+
Raises:
|
|
542
|
+
DatabaseCloneException: If cloning fails
|
|
543
|
+
"""
|
|
544
|
+
# Ensure template database exists
|
|
545
|
+
self._ensure_template_database()
|
|
546
|
+
|
|
547
|
+
# Generate safe worker database name
|
|
548
|
+
worker_db_name = get_safe_worker_database_name(
|
|
549
|
+
self.connection.settings_dict['NAME'],
|
|
550
|
+
worker_id
|
|
551
|
+
)
|
|
552
|
+
|
|
553
|
+
try:
|
|
554
|
+
with self.connection.cursor() as cursor:
|
|
555
|
+
# Check if worker database already exists
|
|
556
|
+
if self.database_exists(worker_db_name):
|
|
557
|
+
logger.warning(f"Database {worker_db_name} already exists, dropping it")
|
|
558
|
+
self.drop_database(worker_db_name)
|
|
559
|
+
|
|
560
|
+
# Create new database
|
|
561
|
+
cursor.execute(f"CREATE DATABASE {worker_db_name}")
|
|
562
|
+
|
|
563
|
+
# Get all tables from template database
|
|
564
|
+
cursor.execute(
|
|
565
|
+
"SELECT table_name FROM information_schema.tables "
|
|
566
|
+
"WHERE table_schema = %s",
|
|
567
|
+
[self._template_db_name]
|
|
568
|
+
)
|
|
569
|
+
tables = [row[0] for row in cursor.fetchall()]
|
|
570
|
+
|
|
571
|
+
# Clone schema for each table with IGNORE DATA
|
|
572
|
+
for table in tables:
|
|
573
|
+
cursor.execute(
|
|
574
|
+
f"CREATE TABLE {worker_db_name}.{table} "
|
|
575
|
+
f"LIKE {self._template_db_name}.{table} IGNORE DATA"
|
|
576
|
+
)
|
|
577
|
+
|
|
578
|
+
logger.info(f"Created MySQL database: {worker_db_name}")
|
|
579
|
+
return worker_db_name
|
|
580
|
+
|
|
581
|
+
except Exception as e:
|
|
582
|
+
raise DatabaseCloneException(
|
|
583
|
+
f"Failed to clone MySQL database: {e}"
|
|
584
|
+
) from e
|
|
585
|
+
|
|
586
|
+
def clone_databases_batch(self, worker_ids: List[int]) -> List[str]:
|
|
587
|
+
"""
|
|
588
|
+
Clone multiple MySQL databases in a single transaction for optimal performance.
|
|
589
|
+
|
|
590
|
+
This method creates multiple worker databases in a single transaction,
|
|
591
|
+
reducing overhead and improving performance for concurrent testing.
|
|
592
|
+
|
|
593
|
+
Args:
|
|
594
|
+
worker_ids (List[int]): List of worker IDs
|
|
595
|
+
|
|
596
|
+
Returns:
|
|
597
|
+
List[str]: List of created database names
|
|
598
|
+
|
|
599
|
+
Raises:
|
|
600
|
+
DatabaseCloneException: If batch cloning fails
|
|
601
|
+
"""
|
|
602
|
+
# Ensure template database exists
|
|
603
|
+
self._ensure_template_database()
|
|
604
|
+
|
|
605
|
+
# Generate all database names
|
|
606
|
+
db_names = [
|
|
607
|
+
get_safe_worker_database_name(
|
|
608
|
+
self.connection.settings_dict['NAME'],
|
|
609
|
+
worker_id
|
|
610
|
+
)
|
|
611
|
+
for worker_id in worker_ids
|
|
612
|
+
]
|
|
613
|
+
|
|
614
|
+
try:
|
|
615
|
+
with self.connection.cursor() as cursor:
|
|
616
|
+
# Start transaction
|
|
617
|
+
cursor.execute("START TRANSACTION")
|
|
618
|
+
|
|
619
|
+
try:
|
|
620
|
+
for db_name in db_names:
|
|
621
|
+
# Check if database already exists
|
|
622
|
+
if self.database_exists(db_name):
|
|
623
|
+
logger.warning(f"Database {db_name} already exists, dropping it")
|
|
624
|
+
self.drop_database(db_name)
|
|
625
|
+
|
|
626
|
+
# Create new database
|
|
627
|
+
cursor.execute(f"CREATE DATABASE {db_name}")
|
|
628
|
+
|
|
629
|
+
# Get all tables from template database
|
|
630
|
+
cursor.execute(
|
|
631
|
+
"SELECT table_name FROM information_schema.tables "
|
|
632
|
+
"WHERE table_schema = %s",
|
|
633
|
+
[self._template_db_name]
|
|
634
|
+
)
|
|
635
|
+
tables = [row[0] for row in cursor.fetchall()]
|
|
636
|
+
|
|
637
|
+
# Clone schema for each table with IGNORE DATA
|
|
638
|
+
for table in tables:
|
|
639
|
+
cursor.execute(
|
|
640
|
+
f"CREATE TABLE {db_name}.{table} "
|
|
641
|
+
f"LIKE {self._template_db_name}.{table} IGNORE DATA"
|
|
642
|
+
)
|
|
643
|
+
|
|
644
|
+
# Commit transaction
|
|
645
|
+
cursor.execute("COMMIT")
|
|
646
|
+
|
|
647
|
+
logger.info(f"Created {len(db_names)} MySQL databases in batch")
|
|
648
|
+
return db_names
|
|
649
|
+
|
|
650
|
+
except Exception as e:
|
|
651
|
+
# Rollback on error
|
|
652
|
+
cursor.execute("ROLLBACK")
|
|
653
|
+
raise e
|
|
654
|
+
|
|
655
|
+
except Exception as e:
|
|
656
|
+
raise DatabaseCloneException(
|
|
657
|
+
f"Failed to clone MySQL databases in batch: {e}"
|
|
658
|
+
) from e
|
|
659
|
+
|
|
660
|
+
def drop_database(self, db_name):
|
|
661
|
+
"""
|
|
662
|
+
Drop MySQL database.
|
|
663
|
+
|
|
664
|
+
Args:
|
|
665
|
+
db_name (str): Database name to drop
|
|
666
|
+
|
|
667
|
+
Raises:
|
|
668
|
+
DatabaseCloneException: If dropping fails
|
|
669
|
+
"""
|
|
670
|
+
from django.db import connection
|
|
671
|
+
|
|
672
|
+
try:
|
|
673
|
+
with connection.cursor() as cursor:
|
|
674
|
+
cursor.execute(f"DROP DATABASE IF EXISTS {db_name}")
|
|
675
|
+
logger.info(f"Dropped MySQL database: {db_name}")
|
|
676
|
+
|
|
677
|
+
except Exception as e:
|
|
678
|
+
raise DatabaseCloneException(
|
|
679
|
+
f"Failed to drop MySQL database {db_name}: {e}"
|
|
680
|
+
) from e
|
|
681
|
+
|
|
682
|
+
def database_exists(self, db_name):
|
|
683
|
+
"""
|
|
684
|
+
Check if MySQL database exists.
|
|
685
|
+
|
|
686
|
+
Args:
|
|
687
|
+
db_name (str): Database name to check
|
|
688
|
+
|
|
689
|
+
Returns:
|
|
690
|
+
bool: True if database exists
|
|
691
|
+
"""
|
|
692
|
+
from django.db import connection
|
|
693
|
+
|
|
694
|
+
try:
|
|
695
|
+
with connection.cursor() as cursor:
|
|
696
|
+
cursor.execute(
|
|
697
|
+
"SELECT 1 FROM information_schema.schemata "
|
|
698
|
+
"WHERE schema_name = %s",
|
|
699
|
+
[db_name]
|
|
700
|
+
)
|
|
701
|
+
return cursor.fetchone() is not None
|
|
702
|
+
except Exception:
|
|
703
|
+
return False
|
|
704
|
+
|
|
705
|
+
|
|
706
|
+
def get_database_cloner(connection):
|
|
707
|
+
"""
|
|
708
|
+
Get appropriate database cloner for connection.
|
|
709
|
+
|
|
710
|
+
Args:
|
|
711
|
+
connection: Django database connection
|
|
712
|
+
|
|
713
|
+
Returns:
|
|
714
|
+
DatabaseCloner: Appropriate cloner instance
|
|
715
|
+
|
|
716
|
+
Raises:
|
|
717
|
+
UnsupportedDatabase: If database backend is not supported
|
|
718
|
+
"""
|
|
719
|
+
vendor = connection.vendor
|
|
720
|
+
|
|
721
|
+
if vendor == 'postgresql':
|
|
722
|
+
return PostgreSQLCloner(connection)
|
|
723
|
+
elif vendor == 'mysql':
|
|
724
|
+
return MySQLCloner(connection)
|
|
725
|
+
elif vendor == 'sqlite':
|
|
726
|
+
return SQLiteCloner(connection)
|
|
727
|
+
else:
|
|
728
|
+
raise UnsupportedDatabase(
|
|
729
|
+
f"Database backend '{vendor}' is not supported. "
|
|
730
|
+
"Only PostgreSQL, MySQL, and SQLite are supported."
|
|
731
|
+
)
|
|
732
|
+
|
|
733
|
+
|
|
734
|
+
def get_thread_safe_connection(alias: str = 'default'):
|
|
735
|
+
"""
|
|
736
|
+
Get a thread-safe database connection for worker operations.
|
|
737
|
+
|
|
738
|
+
This function provides thread-safe access to database connections,
|
|
739
|
+
ensuring that each worker thread has its own isolated connection
|
|
740
|
+
to prevent race conditions and connection sharing issues.
|
|
741
|
+
|
|
742
|
+
Args:
|
|
743
|
+
alias (str): Database alias to connect to
|
|
744
|
+
|
|
745
|
+
Returns:
|
|
746
|
+
DatabaseWrapper: Thread-safe database connection
|
|
747
|
+
|
|
748
|
+
Raises:
|
|
749
|
+
DatabaseCloneException: If connection cannot be established
|
|
750
|
+
"""
|
|
751
|
+
try:
|
|
752
|
+
# Get connection from Django's connection manager
|
|
753
|
+
db_connection = connections[alias]
|
|
754
|
+
|
|
755
|
+
# Ensure connection is established
|
|
756
|
+
db_connection.ensure_connection()
|
|
757
|
+
|
|
758
|
+
return db_connection
|
|
759
|
+
|
|
760
|
+
except Exception as e:
|
|
761
|
+
raise DatabaseCloneException(
|
|
762
|
+
f"Failed to establish thread-safe connection to '{alias}': {e}"
|
|
763
|
+
) from e
|
|
764
|
+
|
|
765
|
+
|
|
766
|
+
def get_worker_connection(worker_id: int, database_name: str, alias: str = 'default'):
|
|
767
|
+
"""
|
|
768
|
+
Get a dedicated database connection for a specific worker.
|
|
769
|
+
|
|
770
|
+
This function creates or retrieves a cached connection for a worker,
|
|
771
|
+
ensuring proper isolation and thread safety. The connection is
|
|
772
|
+
configured to use the worker's specific database.
|
|
773
|
+
|
|
774
|
+
Args:
|
|
775
|
+
worker_id (int): Worker ID
|
|
776
|
+
database_name (str): Database name for the worker
|
|
777
|
+
alias (str): Database alias to use
|
|
778
|
+
|
|
779
|
+
Returns:
|
|
780
|
+
DatabaseWrapper: Worker-specific database connection
|
|
781
|
+
|
|
782
|
+
Raises:
|
|
783
|
+
DatabaseCloneException: If connection cannot be established
|
|
784
|
+
"""
|
|
785
|
+
global _connection_pool, _connection_pool_lock
|
|
786
|
+
|
|
787
|
+
connection_key = f"{worker_id}_{database_name}_{alias}"
|
|
788
|
+
|
|
789
|
+
with _connection_pool_lock:
|
|
790
|
+
if connection_key in _connection_pool:
|
|
791
|
+
# Return cached connection
|
|
792
|
+
cached_connection = _connection_pool[connection_key]
|
|
793
|
+
try:
|
|
794
|
+
# Test if connection is still valid
|
|
795
|
+
cached_connection.cursor().execute("SELECT 1")
|
|
796
|
+
return cached_connection
|
|
797
|
+
except Exception:
|
|
798
|
+
# Connection is stale, remove from pool
|
|
799
|
+
del _connection_pool[connection_key]
|
|
800
|
+
|
|
801
|
+
try:
|
|
802
|
+
# Create new connection for worker
|
|
803
|
+
worker_connection = get_thread_safe_connection(alias)
|
|
804
|
+
|
|
805
|
+
# Configure connection for worker database
|
|
806
|
+
worker_connection.settings_dict = worker_connection.settings_dict.copy()
|
|
807
|
+
worker_connection.settings_dict['NAME'] = database_name
|
|
808
|
+
|
|
809
|
+
# Close existing connection and reconnect to new database
|
|
810
|
+
worker_connection.close()
|
|
811
|
+
worker_connection.ensure_connection()
|
|
812
|
+
|
|
813
|
+
# Cache the connection
|
|
814
|
+
_connection_pool[connection_key] = worker_connection
|
|
815
|
+
|
|
816
|
+
logger.debug(f"Created worker connection for worker {worker_id} to {database_name}")
|
|
817
|
+
return worker_connection
|
|
818
|
+
|
|
819
|
+
except Exception as e:
|
|
820
|
+
raise DatabaseCloneException(
|
|
821
|
+
f"Failed to create worker connection for worker {worker_id}: {e}"
|
|
822
|
+
) from e
|
|
823
|
+
|
|
824
|
+
|
|
825
|
+
def close_worker_connection(worker_id: int, database_name: str, alias: str = 'default'):
|
|
826
|
+
"""
|
|
827
|
+
Close and remove a worker's database connection from the pool.
|
|
828
|
+
|
|
829
|
+
This function properly closes a worker's database connection and
|
|
830
|
+
removes it from the connection pool to prevent connection leaks.
|
|
831
|
+
|
|
832
|
+
Args:
|
|
833
|
+
worker_id (int): Worker ID
|
|
834
|
+
database_name (str): Database name for the worker
|
|
835
|
+
alias (str): Database alias used
|
|
836
|
+
"""
|
|
837
|
+
global _connection_pool, _connection_pool_lock
|
|
838
|
+
|
|
839
|
+
connection_key = f"{worker_id}_{database_name}_{alias}"
|
|
840
|
+
|
|
841
|
+
with _connection_pool_lock:
|
|
842
|
+
if connection_key in _connection_pool:
|
|
843
|
+
try:
|
|
844
|
+
# Close the connection
|
|
845
|
+
_connection_pool[connection_key].close()
|
|
846
|
+
logger.debug(f"Closed worker connection for worker {worker_id}")
|
|
847
|
+
except Exception as e:
|
|
848
|
+
logger.warning(f"Error closing worker connection for worker {worker_id}: {e}")
|
|
849
|
+
finally:
|
|
850
|
+
# Remove from pool
|
|
851
|
+
del _connection_pool[connection_key]
|
|
852
|
+
|
|
853
|
+
|
|
854
|
+
def clear_connection_pool():
|
|
855
|
+
"""
|
|
856
|
+
Clear all connections from the connection pool.
|
|
857
|
+
|
|
858
|
+
This function closes all cached connections and clears the pool,
|
|
859
|
+
typically called during cleanup operations.
|
|
860
|
+
"""
|
|
861
|
+
global _connection_pool, _connection_pool_lock
|
|
862
|
+
|
|
863
|
+
with _connection_pool_lock:
|
|
864
|
+
for connection_key, db_connection in _connection_pool.items():
|
|
865
|
+
try:
|
|
866
|
+
db_connection.close()
|
|
867
|
+
logger.debug(f"Closed connection: {connection_key}")
|
|
868
|
+
except Exception as e:
|
|
869
|
+
logger.warning(f"Error closing connection {connection_key}: {e}")
|
|
870
|
+
|
|
871
|
+
_connection_pool.clear()
|
|
872
|
+
logger.info("Connection pool cleared")
|
|
873
|
+
|
|
874
|
+
|
|
875
|
+
def get_connection_pool_stats():
|
|
876
|
+
"""
|
|
877
|
+
Get statistics about the current connection pool.
|
|
878
|
+
|
|
879
|
+
Returns:
|
|
880
|
+
dict: Connection pool statistics
|
|
881
|
+
"""
|
|
882
|
+
global _connection_pool, _connection_pool_lock
|
|
883
|
+
|
|
884
|
+
with _connection_pool_lock:
|
|
885
|
+
return {
|
|
886
|
+
'total_connections': len(_connection_pool),
|
|
887
|
+
'connection_keys': list(_connection_pool.keys()),
|
|
888
|
+
'pool_size': len(_connection_pool)
|
|
889
|
+
}
|
|
890
|
+
|
|
891
|
+
|
|
892
|
+
@contextmanager
|
|
893
|
+
def worker_database(worker_id, connection):
|
|
894
|
+
"""
|
|
895
|
+
Context manager for worker database operations with thread-safe connections.
|
|
896
|
+
|
|
897
|
+
This context manager provides thread-safe database operations for workers,
|
|
898
|
+
ensuring proper connection isolation and cleanup. It uses the connection
|
|
899
|
+
pool to manage worker-specific connections efficiently.
|
|
900
|
+
|
|
901
|
+
Args:
|
|
902
|
+
worker_id (int): Worker ID
|
|
903
|
+
connection: Django database connection
|
|
904
|
+
|
|
905
|
+
Yields:
|
|
906
|
+
tuple: (worker_database_name, worker_connection)
|
|
907
|
+
|
|
908
|
+
Raises:
|
|
909
|
+
DatabaseCloneException: If database operations fail
|
|
910
|
+
"""
|
|
911
|
+
cloner = get_database_cloner(connection)
|
|
912
|
+
worker_db_name = None
|
|
913
|
+
worker_connection = None
|
|
914
|
+
|
|
915
|
+
try:
|
|
916
|
+
# Clone database for worker
|
|
917
|
+
worker_db_name = cloner.clone_database(worker_id)
|
|
918
|
+
|
|
919
|
+
# Get thread-safe connection for worker
|
|
920
|
+
worker_connection = get_worker_connection(worker_id, worker_db_name)
|
|
921
|
+
|
|
922
|
+
yield (worker_db_name, worker_connection)
|
|
923
|
+
|
|
924
|
+
finally:
|
|
925
|
+
# Clean up worker connection
|
|
926
|
+
if worker_connection and worker_db_name:
|
|
927
|
+
try:
|
|
928
|
+
close_worker_connection(worker_id, worker_db_name)
|
|
929
|
+
except Exception as e:
|
|
930
|
+
logger.warning(f"Failed to close worker connection for worker {worker_id}: {e}")
|
|
931
|
+
|
|
932
|
+
# Clean up worker database
|
|
933
|
+
if worker_db_name and cloner.database_exists(worker_db_name):
|
|
934
|
+
try:
|
|
935
|
+
cloner.drop_database(worker_db_name)
|
|
936
|
+
except Exception as e:
|
|
937
|
+
logger.warning(f"Failed to drop worker database {worker_db_name}: {e}")
|
|
938
|
+
|
|
939
|
+
|
|
940
|
+
@contextmanager
|
|
941
|
+
def worker_database_with_isolation(worker_id, connection):
|
|
942
|
+
"""
|
|
943
|
+
Context manager for worker database operations with isolation verification.
|
|
944
|
+
|
|
945
|
+
This context manager provides the same functionality as worker_database
|
|
946
|
+
but also verifies database isolation to ensure no data leakage between
|
|
947
|
+
workers. This is useful for debugging and ensuring test integrity.
|
|
948
|
+
|
|
949
|
+
Args:
|
|
950
|
+
worker_id (int): Worker ID
|
|
951
|
+
connection: Django database connection
|
|
952
|
+
|
|
953
|
+
Yields:
|
|
954
|
+
tuple: (worker_database_name, worker_connection)
|
|
955
|
+
|
|
956
|
+
Raises:
|
|
957
|
+
DatabaseCloneException: If database operations fail or isolation is violated
|
|
958
|
+
"""
|
|
959
|
+
with worker_database(worker_id, connection) as (worker_db_name, worker_connection):
|
|
960
|
+
# Verify database is ready
|
|
961
|
+
wait_for_database_ready(worker_db_name)
|
|
962
|
+
|
|
963
|
+
# Create a simple isolation test
|
|
964
|
+
try:
|
|
965
|
+
with worker_connection.cursor() as cursor:
|
|
966
|
+
# Create a test table unique to this worker
|
|
967
|
+
cursor.execute(f"""
|
|
968
|
+
CREATE TABLE IF NOT EXISTS worker_{worker_id}_test (
|
|
969
|
+
id SERIAL PRIMARY KEY,
|
|
970
|
+
worker_id INTEGER DEFAULT {worker_id},
|
|
971
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
972
|
+
)
|
|
973
|
+
""")
|
|
974
|
+
|
|
975
|
+
# Insert a test record
|
|
976
|
+
cursor.execute(
|
|
977
|
+
f"INSERT INTO worker_{worker_id}_test (worker_id) VALUES (%s)",
|
|
978
|
+
[worker_id]
|
|
979
|
+
)
|
|
980
|
+
|
|
981
|
+
yield (worker_db_name, worker_connection)
|
|
982
|
+
|
|
983
|
+
finally:
|
|
984
|
+
# Clean up test table
|
|
985
|
+
try:
|
|
986
|
+
with worker_connection.cursor() as cursor:
|
|
987
|
+
cursor.execute(f"DROP TABLE IF EXISTS worker_{worker_id}_test")
|
|
988
|
+
except Exception as e:
|
|
989
|
+
logger.warning(f"Failed to cleanup test table for worker {worker_id}: {e}")
|
|
990
|
+
|
|
991
|
+
|
|
992
|
+
def setup_test_databases(worker_count):
|
|
993
|
+
"""
|
|
994
|
+
Setup test databases for concurrent testing using batch operations.
|
|
995
|
+
|
|
996
|
+
This function uses batch database creation for optimal performance,
|
|
997
|
+
creating all worker databases in a single transaction when possible.
|
|
998
|
+
It also implements template caching to reuse database templates
|
|
999
|
+
across multiple test runs.
|
|
1000
|
+
|
|
1001
|
+
Args:
|
|
1002
|
+
worker_count (int): Number of workers
|
|
1003
|
+
|
|
1004
|
+
Returns:
|
|
1005
|
+
list: List of database names
|
|
1006
|
+
|
|
1007
|
+
Raises:
|
|
1008
|
+
DatabaseCloneException: If setup fails
|
|
1009
|
+
"""
|
|
1010
|
+
# Validate database permissions
|
|
1011
|
+
validate_database_permissions()
|
|
1012
|
+
|
|
1013
|
+
cloner = get_database_cloner(connection)
|
|
1014
|
+
database_names = []
|
|
1015
|
+
|
|
1016
|
+
try:
|
|
1017
|
+
# Use batch cloning if available for better performance
|
|
1018
|
+
if hasattr(cloner, 'clone_databases_batch'):
|
|
1019
|
+
worker_ids = list(range(worker_count))
|
|
1020
|
+
database_names = cloner.clone_databases_batch(worker_ids)
|
|
1021
|
+
logger.info(f"Created {len(database_names)} databases using batch operation")
|
|
1022
|
+
else:
|
|
1023
|
+
# Fallback to individual cloning
|
|
1024
|
+
for worker_id in range(worker_count):
|
|
1025
|
+
db_name = cloner.clone_database(worker_id)
|
|
1026
|
+
database_names.append(db_name)
|
|
1027
|
+
logger.info(f"Created {len(database_names)} databases using individual cloning")
|
|
1028
|
+
|
|
1029
|
+
return database_names
|
|
1030
|
+
|
|
1031
|
+
except Exception as e:
|
|
1032
|
+
# Clean up any created databases
|
|
1033
|
+
for created_db in database_names:
|
|
1034
|
+
try:
|
|
1035
|
+
cloner.drop_database(created_db)
|
|
1036
|
+
except Exception as cleanup_error:
|
|
1037
|
+
logger.warning(f"Failed to cleanup database {created_db}: {cleanup_error}")
|
|
1038
|
+
|
|
1039
|
+
raise DatabaseCloneException(
|
|
1040
|
+
f"Failed to setup test databases: {e}"
|
|
1041
|
+
) from e
|
|
1042
|
+
|
|
1043
|
+
|
|
1044
|
+
def setup_test_databases_with_connections(worker_count):
|
|
1045
|
+
"""
|
|
1046
|
+
Setup test databases and create worker connections.
|
|
1047
|
+
|
|
1048
|
+
This function sets up test databases and creates dedicated
|
|
1049
|
+
connections for each worker, ensuring proper isolation and
|
|
1050
|
+
thread safety.
|
|
1051
|
+
|
|
1052
|
+
Args:
|
|
1053
|
+
worker_count (int): Number of workers
|
|
1054
|
+
|
|
1055
|
+
Returns:
|
|
1056
|
+
dict: Mapping of worker_id to (database_name, connection)
|
|
1057
|
+
|
|
1058
|
+
Raises:
|
|
1059
|
+
DatabaseCloneException: If setup fails
|
|
1060
|
+
"""
|
|
1061
|
+
# Setup databases
|
|
1062
|
+
database_names = setup_test_databases(worker_count)
|
|
1063
|
+
|
|
1064
|
+
# Create worker connections
|
|
1065
|
+
worker_connections = {}
|
|
1066
|
+
|
|
1067
|
+
try:
|
|
1068
|
+
for worker_id, database_name in enumerate(database_names):
|
|
1069
|
+
worker_connection = get_worker_connection(worker_id, database_name)
|
|
1070
|
+
worker_connections[worker_id] = (database_name, worker_connection)
|
|
1071
|
+
|
|
1072
|
+
logger.info(f"Created {len(worker_connections)} worker connections")
|
|
1073
|
+
return worker_connections
|
|
1074
|
+
|
|
1075
|
+
except Exception as e:
|
|
1076
|
+
# Cleanup on error
|
|
1077
|
+
for worker_id, (database_name, conn) in worker_connections.items():
|
|
1078
|
+
try:
|
|
1079
|
+
close_worker_connection(worker_id, database_name)
|
|
1080
|
+
except Exception:
|
|
1081
|
+
pass
|
|
1082
|
+
|
|
1083
|
+
# Also cleanup databases
|
|
1084
|
+
cloner = get_database_cloner(connection)
|
|
1085
|
+
for database_name in database_names:
|
|
1086
|
+
try:
|
|
1087
|
+
cloner.drop_database(database_name)
|
|
1088
|
+
except Exception:
|
|
1089
|
+
pass
|
|
1090
|
+
|
|
1091
|
+
raise DatabaseCloneException(
|
|
1092
|
+
f"Failed to setup test databases with connections: {e}"
|
|
1093
|
+
) from e
|
|
1094
|
+
|
|
1095
|
+
|
|
1096
|
+
def teardown_test_databases(database_names):
|
|
1097
|
+
"""
|
|
1098
|
+
Teardown test databases with connection cleanup.
|
|
1099
|
+
|
|
1100
|
+
This function drops test databases and cleans up any associated
|
|
1101
|
+
connections from the connection pool to prevent resource leaks.
|
|
1102
|
+
|
|
1103
|
+
Args:
|
|
1104
|
+
database_names (list): List of database names to drop
|
|
1105
|
+
|
|
1106
|
+
Raises:
|
|
1107
|
+
DatabaseCloneException: If teardown fails
|
|
1108
|
+
"""
|
|
1109
|
+
cloner = get_database_cloner(connection)
|
|
1110
|
+
|
|
1111
|
+
# Close any worker connections to these databases
|
|
1112
|
+
pool_stats = get_connection_pool_stats()
|
|
1113
|
+
for connection_key in pool_stats['connection_keys']:
|
|
1114
|
+
# Parse connection key to get worker_id and database_name
|
|
1115
|
+
parts = connection_key.split('_')
|
|
1116
|
+
if len(parts) >= 3:
|
|
1117
|
+
worker_id = int(parts[0])
|
|
1118
|
+
database_name = '_'.join(parts[1:-1]) # Everything except first and last parts
|
|
1119
|
+
if database_name in database_names:
|
|
1120
|
+
close_worker_connection(worker_id, database_name)
|
|
1121
|
+
|
|
1122
|
+
# Drop databases
|
|
1123
|
+
for db_name in database_names:
|
|
1124
|
+
try:
|
|
1125
|
+
if cloner.database_exists(db_name):
|
|
1126
|
+
cloner.drop_database(db_name)
|
|
1127
|
+
logger.debug(f"Dropped test database: {db_name}")
|
|
1128
|
+
except Exception as e:
|
|
1129
|
+
logger.warning(f"Failed to drop test database {db_name}: {e}")
|
|
1130
|
+
|
|
1131
|
+
|
|
1132
|
+
def teardown_test_databases_with_connections(worker_connections):
|
|
1133
|
+
"""
|
|
1134
|
+
Teardown test databases and close worker connections.
|
|
1135
|
+
|
|
1136
|
+
This function properly cleans up both databases and their associated
|
|
1137
|
+
connections, ensuring complete resource cleanup.
|
|
1138
|
+
|
|
1139
|
+
Args:
|
|
1140
|
+
worker_connections (dict): Mapping of worker_id to (database_name, connection)
|
|
1141
|
+
|
|
1142
|
+
Raises:
|
|
1143
|
+
DatabaseCloneException: If teardown fails
|
|
1144
|
+
"""
|
|
1145
|
+
# Close all worker connections
|
|
1146
|
+
for worker_id, (database_name, conn) in worker_connections.items():
|
|
1147
|
+
try:
|
|
1148
|
+
close_worker_connection(worker_id, database_name)
|
|
1149
|
+
except Exception as e:
|
|
1150
|
+
logger.warning(f"Failed to close worker connection for worker {worker_id}: {e}")
|
|
1151
|
+
|
|
1152
|
+
# Drop databases
|
|
1153
|
+
database_names = [db_name for _, (db_name, _) in worker_connections.items()]
|
|
1154
|
+
teardown_test_databases(database_names)
|
|
1155
|
+
|
|
1156
|
+
|
|
1157
|
+
def wait_for_database_ready(db_name, timeout=30):
|
|
1158
|
+
"""
|
|
1159
|
+
Wait for database to be ready with thread-safe connection.
|
|
1160
|
+
|
|
1161
|
+
This function uses thread-safe connections to test database readiness,
|
|
1162
|
+
ensuring proper isolation and preventing connection conflicts.
|
|
1163
|
+
|
|
1164
|
+
Args:
|
|
1165
|
+
db_name (str): Database name
|
|
1166
|
+
timeout (int): Timeout in seconds
|
|
1167
|
+
|
|
1168
|
+
Returns:
|
|
1169
|
+
bool: True if database is ready
|
|
1170
|
+
|
|
1171
|
+
Raises:
|
|
1172
|
+
DatabaseCloneException: If database is not ready within timeout
|
|
1173
|
+
"""
|
|
1174
|
+
cloner = get_database_cloner(connection)
|
|
1175
|
+
start_time = time.time()
|
|
1176
|
+
|
|
1177
|
+
while time.time() - start_time < timeout:
|
|
1178
|
+
if cloner.database_exists(db_name):
|
|
1179
|
+
try:
|
|
1180
|
+
# Use thread-safe connection to test database
|
|
1181
|
+
test_connection = get_thread_safe_connection()
|
|
1182
|
+
test_connection.settings_dict = test_connection.settings_dict.copy()
|
|
1183
|
+
test_connection.settings_dict['NAME'] = db_name
|
|
1184
|
+
|
|
1185
|
+
# Test connection to the database
|
|
1186
|
+
with test_connection.cursor() as cursor:
|
|
1187
|
+
cursor.execute("SELECT 1")
|
|
1188
|
+
|
|
1189
|
+
# Close test connection
|
|
1190
|
+
test_connection.close()
|
|
1191
|
+
return True
|
|
1192
|
+
|
|
1193
|
+
except Exception:
|
|
1194
|
+
pass
|
|
1195
|
+
|
|
1196
|
+
time.sleep(0.1)
|
|
1197
|
+
|
|
1198
|
+
raise DatabaseCloneException(
|
|
1199
|
+
f"Database {db_name} not ready within {timeout} seconds"
|
|
1200
|
+
)
|
|
1201
|
+
|
|
1202
|
+
|
|
1203
|
+
def terminate_connections(db_name, vendor):
|
|
1204
|
+
"""
|
|
1205
|
+
Terminate all connections to a database.
|
|
1206
|
+
|
|
1207
|
+
Args:
|
|
1208
|
+
db_name (str): Database name
|
|
1209
|
+
vendor (str): Database vendor ('postgresql', 'mysql', etc.)
|
|
1210
|
+
"""
|
|
1211
|
+
try:
|
|
1212
|
+
with connection.cursor() as cursor:
|
|
1213
|
+
if vendor == 'postgresql':
|
|
1214
|
+
# PostgreSQL implementation
|
|
1215
|
+
cursor.execute(f"""
|
|
1216
|
+
SELECT pg_terminate_backend(pid)
|
|
1217
|
+
FROM pg_stat_activity
|
|
1218
|
+
WHERE datname = '{db_name}'
|
|
1219
|
+
AND pid <> pg_backend_pid()
|
|
1220
|
+
""")
|
|
1221
|
+
elif vendor == 'mysql':
|
|
1222
|
+
# MySQL implementation
|
|
1223
|
+
cursor.execute(f"""
|
|
1224
|
+
SELECT CONCAT('KILL ', id, ';')
|
|
1225
|
+
FROM information_schema.processlist
|
|
1226
|
+
WHERE db = '{db_name}'
|
|
1227
|
+
AND id <> CONNECTION_ID()
|
|
1228
|
+
""")
|
|
1229
|
+
results = cursor.fetchall()
|
|
1230
|
+
for result in results:
|
|
1231
|
+
if result[0]:
|
|
1232
|
+
cursor.execute(result[0])
|
|
1233
|
+
elif vendor == 'sqlite3':
|
|
1234
|
+
# No-op for SQLite - file-based database doesn't need connection termination
|
|
1235
|
+
logger.debug(f"SQLite3: Skipping connection termination for {db_name}")
|
|
1236
|
+
# Add other database support as needed
|
|
1237
|
+
else:
|
|
1238
|
+
logger.warning(f"Terminate connections not implemented for vendor: {vendor}")
|
|
1239
|
+
|
|
1240
|
+
except Exception as e:
|
|
1241
|
+
logger.warning(f"Failed to terminate connections for {db_name}: {e}")
|
|
1242
|
+
|
|
1243
|
+
|
|
1244
|
+
def verify_database_isolation(worker_connections):
|
|
1245
|
+
"""
|
|
1246
|
+
Verify that worker databases are properly isolated.
|
|
1247
|
+
|
|
1248
|
+
This function tests that each worker database is isolated from others
|
|
1249
|
+
by creating test data in one database and verifying it doesn't appear
|
|
1250
|
+
in other databases.
|
|
1251
|
+
|
|
1252
|
+
Args:
|
|
1253
|
+
worker_connections (dict): Mapping of worker_id to (database_name, connection)
|
|
1254
|
+
|
|
1255
|
+
Returns:
|
|
1256
|
+
bool: True if all databases are properly isolated
|
|
1257
|
+
|
|
1258
|
+
Raises:
|
|
1259
|
+
DatabaseCloneException: If isolation verification fails
|
|
1260
|
+
"""
|
|
1261
|
+
if len(worker_connections) < 2:
|
|
1262
|
+
# Need at least 2 databases to test isolation
|
|
1263
|
+
return True
|
|
1264
|
+
|
|
1265
|
+
try:
|
|
1266
|
+
# Create test data in first worker database
|
|
1267
|
+
first_worker_id = min(worker_connections.keys())
|
|
1268
|
+
first_db_name, first_connection = worker_connections[first_worker_id]
|
|
1269
|
+
|
|
1270
|
+
# Create a test table and insert data
|
|
1271
|
+
with first_connection.cursor() as cursor:
|
|
1272
|
+
cursor.execute("""
|
|
1273
|
+
CREATE TABLE IF NOT EXISTS isolation_test (
|
|
1274
|
+
id SERIAL PRIMARY KEY,
|
|
1275
|
+
worker_id INTEGER,
|
|
1276
|
+
test_data TEXT
|
|
1277
|
+
)
|
|
1278
|
+
""")
|
|
1279
|
+
cursor.execute(
|
|
1280
|
+
"INSERT INTO isolation_test (worker_id, test_data) VALUES (%s, %s)",
|
|
1281
|
+
[first_worker_id, f"test_data_from_worker_{first_worker_id}"]
|
|
1282
|
+
)
|
|
1283
|
+
|
|
1284
|
+
# Verify data doesn't exist in other databases
|
|
1285
|
+
for worker_id, (db_name, conn) in worker_connections.items():
|
|
1286
|
+
if worker_id == first_worker_id:
|
|
1287
|
+
continue
|
|
1288
|
+
|
|
1289
|
+
with conn.cursor() as cursor:
|
|
1290
|
+
cursor.execute("SELECT COUNT(*) FROM isolation_test")
|
|
1291
|
+
count = cursor.fetchone()[0]
|
|
1292
|
+
|
|
1293
|
+
if count > 0:
|
|
1294
|
+
raise DatabaseCloneException(
|
|
1295
|
+
f"Database isolation failed: Worker {worker_id} database "
|
|
1296
|
+
f"contains data from worker {first_worker_id}"
|
|
1297
|
+
)
|
|
1298
|
+
|
|
1299
|
+
# Clean up test data
|
|
1300
|
+
with first_connection.cursor() as cursor:
|
|
1301
|
+
cursor.execute("DROP TABLE IF EXISTS isolation_test")
|
|
1302
|
+
|
|
1303
|
+
logger.info("Database isolation verification passed")
|
|
1304
|
+
return True
|
|
1305
|
+
|
|
1306
|
+
except Exception as e:
|
|
1307
|
+
raise DatabaseCloneException(
|
|
1308
|
+
f"Database isolation verification failed: {e}"
|
|
1309
|
+
) from e
|