bauta 0.2.3__tar.gz → 0.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {bauta-0.2.3 → bauta-0.2.4}/PKG-INFO +3 -3
- {bauta-0.2.3 → bauta-0.2.4}/bauta/cli/__init__.py +15 -2
- {bauta-0.2.3 → bauta-0.2.4}/bauta/cli/common.py +19 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/cli/review.py +27 -6
- {bauta-0.2.3 → bauta-0.2.4}/bauta/cli/run.py +12 -1
- {bauta-0.2.3 → bauta-0.2.4}/bauta/configuration/__init__.py +4 -2
- {bauta-0.2.3 → bauta-0.2.4}/bauta/configuration/models.py +87 -5
- {bauta-0.2.3 → bauta-0.2.4}/bauta/database/connection.py +51 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/database/dialects/base.py +82 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/database/dialects/duckdb.py +41 -1
- {bauta-0.2.3 → bauta-0.2.4}/bauta/database/dialects/mssql.py +55 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/database/dialects/mysql.py +11 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/database/dialects/oracle.py +14 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/database/dialects/postgresql.py +18 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/database/dialects/sqlite.py +60 -3
- {bauta-0.2.3 → bauta-0.2.4}/bauta/generate/builtinDiscovery.py +9 -4
- {bauta-0.2.3 → bauta-0.2.4}/bauta/generate/discovery.py +70 -9
- {bauta-0.2.3 → bauta-0.2.4}/bauta/generate/synthesize.py +4 -2
- {bauta-0.2.3 → bauta-0.2.4}/bauta/jobs/__init__.py +2 -1
- {bauta-0.2.3 → bauta-0.2.4}/bauta/jobs/dependencyGraph.py +46 -2
- {bauta-0.2.3 → bauta-0.2.4}/bauta/jobs/keys.py +39 -18
- bauta-0.2.4/bauta/jobs/partitions.py +193 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/jobs/pipeline.py +292 -36
- {bauta-0.2.3 → bauta-0.2.4}/bauta/jobs/runner.py +28 -2
- {bauta-0.2.3 → bauta-0.2.4}/bauta/jobs/targets.py +70 -5
- {bauta-0.2.3 → bauta-0.2.4}/bauta/jobs/workers.py +8 -4
- {bauta-0.2.3 → bauta-0.2.4}/bauta/lake/iceberg.py +2 -1
- {bauta-0.2.3 → bauta-0.2.4}/bauta/lake/target.py +2 -1
- {bauta-0.2.3 → bauta-0.2.4}/bauta/log/__init__.py +4 -1
- {bauta-0.2.3 → bauta-0.2.4}/bauta/log/core.py +58 -4
- bauta-0.2.4/bauta/log/scrubbing.py +258 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/masking/__init__.py +11 -4
- {bauta-0.2.3 → bauta-0.2.4}/bauta/masking/core.py +225 -14
- bauta-0.2.4/bauta/masking/fakeData.py +563 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/masking/strategies.py +390 -27
- {bauta-0.2.3 → bauta-0.2.4}/bauta/review/audit.py +147 -18
- bauta-0.2.4/bauta/review/html.py +259 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta.egg-info/PKG-INFO +3 -3
- {bauta-0.2.3 → bauta-0.2.4}/bauta.egg-info/SOURCES.txt +2 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta.egg-info/requires.txt +2 -2
- {bauta-0.2.3 → bauta-0.2.4}/pyproject.toml +7 -4
- bauta-0.2.3/bauta/log/scrubbing.py +0 -115
- bauta-0.2.3/bauta/masking/fakeData.py +0 -162
- {bauta-0.2.3 → bauta-0.2.4}/LICENSE +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/README.md +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/__init__.py +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/__main__.py +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/cli/generate.py +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/configuration/columnTypes.py +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/configuration/connections.py +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/configuration/environment.py +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/configuration/files.py +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/database/__init__.py +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/database/dialects/__init__.py +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/database/dialects/names.py +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/database/driver.py +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/database/values.py +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/generate/__init__.py +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/generate/drift.py +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/generate/schema.py +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/generate/subset.py +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/jobs/fullRefresh.py +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/jobs/memory.py +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/jobs/reporting.py +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/lake/__init__.py +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/lake/columnar.py +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/lake/columns.py +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/lake/formats.py +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/lake/stores.py +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/masking/fpe.py +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/py.typed +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/review/__init__.py +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/review/coverage.py +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/review/references.py +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/transform/__init__.py +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/transform/builtinTransforms.py +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta/transform/core.py +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta.egg-info/dependency_links.txt +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta.egg-info/entry_points.txt +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/bauta.egg-info/top_level.txt +0 -0
- {bauta-0.2.3 → bauta-0.2.4}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: bauta
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.4
|
|
4
4
|
Summary: Safe, realistic copies of production across databases: masking, subsets, synthetic data, and the ETL to move them
|
|
5
5
|
Author: David Ribeiro
|
|
6
6
|
License-Expression: MIT
|
|
@@ -42,7 +42,7 @@ Provides-Extra: mariadb
|
|
|
42
42
|
Requires-Dist: mysql-connector-python>=8.4; extra == "mariadb"
|
|
43
43
|
Provides-Extra: sqlite
|
|
44
44
|
Provides-Extra: duckdb
|
|
45
|
-
Requires-Dist: duckdb>=1.
|
|
45
|
+
Requires-Dist: duckdb>=1.5; extra == "duckdb"
|
|
46
46
|
Requires-Dist: pyarrow>=21; extra == "duckdb"
|
|
47
47
|
Requires-Dist: pytz>=2024.1; extra == "duckdb"
|
|
48
48
|
Provides-Extra: files
|
|
@@ -55,7 +55,7 @@ Requires-Dist: cryptography>=42; extra == "fpe"
|
|
|
55
55
|
Provides-Extra: all
|
|
56
56
|
Requires-Dist: bauta[duckdb,files,fpe,iceberg,mariadb,mssql,mysql,oracle,postgresql,sqlite]; extra == "all"
|
|
57
57
|
Provides-Extra: native
|
|
58
|
-
Requires-Dist: bauta-rs==0.2.
|
|
58
|
+
Requires-Dist: bauta-rs==0.2.4; extra == "native"
|
|
59
59
|
Provides-Extra: docs
|
|
60
60
|
Requires-Dist: mkdocs==1.6.1; extra == "docs"
|
|
61
61
|
Requires-Dist: mkdocs-material==9.7.7; extra == "docs"
|
|
@@ -89,6 +89,15 @@ def _positiveInteger(text: str) -> int:
|
|
|
89
89
|
return value
|
|
90
90
|
|
|
91
91
|
|
|
92
|
+
def _nonNegativeInteger(text: str) -> int:
|
|
93
|
+
|
|
94
|
+
value = int(text)
|
|
95
|
+
if value < 0:
|
|
96
|
+
raise argparse.ArgumentTypeError('must be at least 0, got {}'.format(value))
|
|
97
|
+
|
|
98
|
+
return value
|
|
99
|
+
|
|
100
|
+
|
|
92
101
|
def _addRunArguments(parser: argparse.ArgumentParser) -> None:
|
|
93
102
|
|
|
94
103
|
parser.add_argument('--forever', action='store_true',
|
|
@@ -179,7 +188,10 @@ def _buildParser() -> argparse.ArgumentParser:
|
|
|
179
188
|
auditParser.add_argument('--connect', action='store_true',
|
|
180
189
|
help='also run each masked query for its real columns, and check whether each connection is encrypted')
|
|
181
190
|
auditParser.add_argument('--job', action='append', help='audit only this job (repeatable)')
|
|
182
|
-
auditParser.add_argument('--
|
|
191
|
+
auditParser.add_argument('--sample', type=_nonNegativeInteger, default=1000, metavar='ROWS',
|
|
192
|
+
help='with --connect, rows of each query read to check kept columns and shared domains by their values; 0 reads none')
|
|
193
|
+
auditParser.add_argument('--format', default='text', choices=['text', 'json', 'html'],
|
|
194
|
+
help='text, json, or html: one self-contained page for a reviewer, best with --output (default: text)')
|
|
183
195
|
auditParser.add_argument('--strict', action='store_true', help='exit 1 on warnings as well as errors')
|
|
184
196
|
auditParser.add_argument('--output', help='write the report here instead of stdout; must not already exist')
|
|
185
197
|
_addRulesArgument(auditParser)
|
|
@@ -198,7 +210,8 @@ def _buildParser() -> argparse.ArgumentParser:
|
|
|
198
210
|
coverageParser.add_argument('--connection', help='the source connection alias to check; required when the jobs read from more than one')
|
|
199
211
|
coverageParser.add_argument('--schema', help='the schema to list, instead of the connection\'s own')
|
|
200
212
|
coverageParser.add_argument('--job', action='append', help='only these jobs count as covering a table. Repeatable.')
|
|
201
|
-
coverageParser.add_argument('--format', choices=['text', 'json'], default='text',
|
|
213
|
+
coverageParser.add_argument('--format', choices=['text', 'json', 'html'], default='text',
|
|
214
|
+
help='text, json, or html: one self-contained page for a reviewer, best with --output (default: text)')
|
|
202
215
|
coverageParser.add_argument('--output', help='write to this file instead of stdout')
|
|
203
216
|
|
|
204
217
|
verifyParser = subparsers.add_parser('verify-manifest', help='check that a manifest is unaltered, and who signed it')
|
|
@@ -300,6 +300,25 @@ def _sourceQueryColumns(job: DataJobConfig, connections: _Connections) -> List[s
|
|
|
300
300
|
return columns
|
|
301
301
|
|
|
302
302
|
|
|
303
|
+
def _sourceQuerySample(job: DataJobConfig, connections: _Connections, rows: int) -> Tuple[List[str], List[Tuple[Any, ...]]]:
|
|
304
|
+
"""The columns a job's sourceQuery returns, and up to `rows` of its rows
|
|
305
|
+
from the start, without reading the rest. They stay in memory: nothing
|
|
306
|
+
that reads them may report a value.
|
|
307
|
+
"""
|
|
308
|
+
|
|
309
|
+
with connections.use(job.sourceConnection) as database:
|
|
310
|
+
query = job.sourceQuery
|
|
311
|
+
parameters = None
|
|
312
|
+
if job.watermarkColumn:
|
|
313
|
+
query = database.substituteWatermarkPlaceholder(query)
|
|
314
|
+
parameters = (job.watermarkInitial,)
|
|
315
|
+
columns, chunks = database.stream(query=query, chunkSize=max(1, rows), parameters=parameters)
|
|
316
|
+
with chunks:
|
|
317
|
+
sampled = next(chunks, []) if rows else []
|
|
318
|
+
|
|
319
|
+
return columns, list(sampled)
|
|
320
|
+
|
|
321
|
+
|
|
303
322
|
def _targetColumns(job: DataJobConfig, connections: _Connections) -> List[str]:
|
|
304
323
|
"""The target's columns, in the order a load fills them."""
|
|
305
324
|
|
|
@@ -13,7 +13,7 @@ from ..database.dialects import ForeignKey, bareName, splitTableName, unqualifie
|
|
|
13
13
|
from ..log import Log
|
|
14
14
|
from ..log.scrubbing import describeError
|
|
15
15
|
from .common import (EXIT_JOBS_DID_NOT_SUCCEED, EXIT_SUCCESS, _Connections, UsageError, _discoveryRules, _loadDataJobs, _requireDatabase, _selectJobs,
|
|
16
|
-
|
|
16
|
+
_sourceQuerySample, _targetColumns, _toolVersion, _writeOutput)
|
|
17
17
|
|
|
18
18
|
|
|
19
19
|
def _commandVerifyReferences(arguments: argparse.Namespace, log: Log) -> int:
|
|
@@ -136,8 +136,16 @@ def _commandCoverage(arguments: argparse.Namespace, log: Log) -> int:
|
|
|
136
136
|
|
|
137
137
|
report = coverageReport(alias, tables, jobs, acknowledged=jobsFile.acknowledged.get(alias),
|
|
138
138
|
columns=columns, rules=_discoveryRules(arguments))
|
|
139
|
-
|
|
140
|
-
|
|
139
|
+
if arguments.format == 'html':
|
|
140
|
+
import datetime
|
|
141
|
+
|
|
142
|
+
from ..review.html import renderCoverageHtml
|
|
143
|
+
|
|
144
|
+
generatedAt = datetime.datetime.now(datetime.timezone.utc).isoformat(timespec='seconds')
|
|
145
|
+
text = renderCoverageHtml(report, generatedAt=generatedAt, version=_toolVersion())
|
|
146
|
+
else:
|
|
147
|
+
text = json.dumps(report, indent=2, default=str) + '\n' if arguments.format == 'json' else renderCoverage(report)
|
|
148
|
+
_writeOutput(text, arguments.output)
|
|
141
149
|
|
|
142
150
|
if report['summary'][UNCOVERED]:
|
|
143
151
|
return EXIT_JOBS_DID_NOT_SUCCEED
|
|
@@ -162,6 +170,11 @@ def _commandAudit(arguments: argparse.Namespace, log: Log) -> int:
|
|
|
162
170
|
question. Offline unless --connect, which also resolves each masked
|
|
163
171
|
query's real columns and checks whether each connection is encrypted.
|
|
164
172
|
|
|
173
|
+
With --connect it also reads up to --sample rows of each query, to
|
|
174
|
+
question a column kept as it is whose values look like personal data, and
|
|
175
|
+
columns sharing a domain that hold its values differently. The rows stay
|
|
176
|
+
in memory, and no value is reported.
|
|
177
|
+
|
|
165
178
|
Exits 1 on an error finding, and with --strict on a warning too.
|
|
166
179
|
"""
|
|
167
180
|
|
|
@@ -177,6 +190,7 @@ def _commandAudit(arguments: argparse.Namespace, log: Log) -> int:
|
|
|
177
190
|
foreignKeys: Dict[str, List[ForeignKey]] = {}
|
|
178
191
|
declaredForeignKeys: Dict[str, List[ForeignKey]] = {}
|
|
179
192
|
elsewhere: Dict[str, Dict[str, int]] = {}
|
|
193
|
+
samples: Dict[str, List[Tuple[Any, ...]]] = {}
|
|
180
194
|
|
|
181
195
|
if arguments.connect:
|
|
182
196
|
# Every job's columns, not only a masked one's: an unmasked job's are
|
|
@@ -186,7 +200,7 @@ def _commandAudit(arguments: argparse.Namespace, log: Log) -> int:
|
|
|
186
200
|
with _Connections(connectionConfiguration) as connections:
|
|
187
201
|
for name, job in jobs.items():
|
|
188
202
|
try:
|
|
189
|
-
returnedColumns[name] =
|
|
203
|
+
returnedColumns[name], samples[name] = _sourceQuerySample(job, connections, arguments.sample)
|
|
190
204
|
except Exception as error:
|
|
191
205
|
unreachable[name] = describeError(error)
|
|
192
206
|
continue
|
|
@@ -242,8 +256,15 @@ def _commandAudit(arguments: argparse.Namespace, log: Log) -> int:
|
|
|
242
256
|
report = auditJobs(jobs, returnedColumns=returnedColumns, encryption=encryption, unreachable=unreachable,
|
|
243
257
|
targetColumns=targetColumns, foreignKeys=foreignKeys, declaredForeignKeys=declaredForeignKeys,
|
|
244
258
|
rules=_discoveryRules(arguments), connections=connectionConfiguration,
|
|
245
|
-
foreignKeysElsewhere=elsewhere if arguments.connect else None, unreadableTargets=unreadableTargets
|
|
246
|
-
|
|
259
|
+
foreignKeysElsewhere=elsewhere if arguments.connect else None, unreadableTargets=unreadableTargets,
|
|
260
|
+
samples=samples)
|
|
261
|
+
if arguments.format == 'html':
|
|
262
|
+
from ..review.html import renderAuditHtml
|
|
263
|
+
|
|
264
|
+
text = renderAuditHtml(report, version=_toolVersion())
|
|
265
|
+
else:
|
|
266
|
+
text = json.dumps(report, indent=2, default=str) + '\n' if arguments.format == 'json' else renderAudit(report)
|
|
267
|
+
_writeOutput(text, arguments.output)
|
|
247
268
|
|
|
248
269
|
if report['summary']['error'] or (arguments.strict and report['summary']['warning']):
|
|
249
270
|
return EXIT_JOBS_DID_NOT_SUCCEED
|
|
@@ -276,12 +276,16 @@ def _commandValidate(arguments: argparse.Namespace, log: Log) -> int:
|
|
|
276
276
|
except ConfigurationError as error:
|
|
277
277
|
problems.append('{}: {}'.format(alias, error))
|
|
278
278
|
|
|
279
|
-
from ..masking import effectiveMaskingThreads
|
|
279
|
+
from ..masking import effectiveMaskingThreads, requireNativeProblem
|
|
280
280
|
|
|
281
281
|
try:
|
|
282
282
|
effectiveMaskingThreads(jobsFile.maskingThreads)
|
|
283
283
|
except ValueError as error:
|
|
284
284
|
problems.append(str(error))
|
|
285
|
+
if any(job.masking is not None and job.active for job in jobsFile.jobs.values()):
|
|
286
|
+
problem = requireNativeProblem(jobsFile.requireNative)
|
|
287
|
+
if problem:
|
|
288
|
+
problems.append(problem)
|
|
285
289
|
|
|
286
290
|
if problems:
|
|
287
291
|
raise ConfigurationError('invalid configuration:\n' + '\n'.join(problems))
|
|
@@ -414,6 +418,13 @@ def _dryRunDataJobs(jobsFile: DataJobsFile, connectionConfiguration: Dict[str, C
|
|
|
414
418
|
problem = _checkColumnCounts(name, job, returned, returned if columns is None else columns)
|
|
415
419
|
if problem is None and job.masking is not None:
|
|
416
420
|
problem = _checkMaskingCoverage(name, job, returned, log)
|
|
421
|
+
if problem is None and job.partitions is not None and job.partitions.column is not None:
|
|
422
|
+
from ..jobs.partitions import resolveColumn
|
|
423
|
+
|
|
424
|
+
try:
|
|
425
|
+
resolveColumn(job.partitions.column, returned)
|
|
426
|
+
except ConfigurationError as error:
|
|
427
|
+
problem = '{}: {}'.format(name, error)
|
|
417
428
|
if problem:
|
|
418
429
|
problems.append(problem)
|
|
419
430
|
|
|
@@ -8,8 +8,8 @@ from .connections import (EMBEDDED_TYPES, IDENTIFIER, S3_LARGEST_COPY, Connectio
|
|
|
8
8
|
FileCompression, FileFormat, FilesConnection, FileStore, IcebergCatalog, IcebergConnection, LakeConfig, MariaDBConnection, anchorPaths, isSchemePath,
|
|
9
9
|
MSSQLConnection, MySQLConnection, OracleConnection, PostgreSQLConnection, SQLiteConnection, StoreType)
|
|
10
10
|
from .models import (FILE_STRATEGIES, WATERMARK_PLACEHOLDER, BaseJobConfig, Configuration, DataJobConfig, DataJobsFile, DiscoveryRulesFile,
|
|
11
|
-
InsertStrategy, MaskingConfig, NameRuleConfig, StorageLocation, TableLocation, ValueRuleConfig, connectionConfig, filePathProblem,
|
|
12
|
-
findCycle, isLake, targetKind, targetProblems)
|
|
11
|
+
InsertStrategy, MaskingConfig, NameRuleConfig, PartitionsConfig, StorageLocation, TableLocation, ValueRuleConfig, connectionConfig, filePathProblem,
|
|
12
|
+
findCycle, isLake, partitionLimitProblems, targetKind, targetProblems)
|
|
13
13
|
from .columnTypes import ColumnType, parseColumnType
|
|
14
14
|
from .files import INCLUDE_KEY, JobsDocument, loadYamlFile, readJobsFile
|
|
15
15
|
|
|
@@ -53,6 +53,8 @@ __all__ = [
|
|
|
53
53
|
'MySQLConnection',
|
|
54
54
|
'NameRuleConfig',
|
|
55
55
|
'OracleConnection',
|
|
56
|
+
'partitionLimitProblems',
|
|
57
|
+
'PartitionsConfig',
|
|
56
58
|
'parseColumnType',
|
|
57
59
|
'PasswordCommandError',
|
|
58
60
|
'PostgreSQLConnection',
|
|
@@ -11,7 +11,7 @@ from typing import Annotated, Any, Callable, Dict, List, Literal, Mapping, Optio
|
|
|
11
11
|
|
|
12
12
|
from pydantic import BaseModel, ConfigDict, Field, SecretStr, ValidationError, field_validator, model_validator
|
|
13
13
|
|
|
14
|
-
from ..masking import changesValues, policyFor, validateColumnPolicy, validateKey
|
|
14
|
+
from ..masking import changesValues, policyFor, validateColumnPolicy, validateKey, warnIfWeakKey
|
|
15
15
|
from .connections import (_CONNECTION_ADAPTER, CONNECTION_TYPES, anchorPaths, CleanedListMapping, CleanedMapping, CleanedStringList, ConnectionConfig,
|
|
16
16
|
DuckDBConnection, FilesConnection, IcebergConnection, _listed)
|
|
17
17
|
from .environment import ConfigurationError
|
|
@@ -127,6 +127,7 @@ class MaskingConfig(BaseModel):
|
|
|
127
127
|
def _requireStrongKey(cls, key: SecretStr) -> SecretStr:
|
|
128
128
|
|
|
129
129
|
validateKey(key.get_secret_value())
|
|
130
|
+
warnIfWeakKey(key.get_secret_value())
|
|
130
131
|
|
|
131
132
|
return key
|
|
132
133
|
|
|
@@ -145,6 +146,44 @@ class MaskingConfig(BaseModel):
|
|
|
145
146
|
return None if policy is None else validateColumnPolicy(policy)
|
|
146
147
|
|
|
147
148
|
|
|
149
|
+
class PartitionsConfig(BaseModel):
|
|
150
|
+
"""A job read, masked and written as `count` slices at once, each a range
|
|
151
|
+
of the numeric `column` its sourceQuery returns. See "Partitions" in
|
|
152
|
+
docs/concepts/how-it-works.md.
|
|
153
|
+
|
|
154
|
+
At least two: one slice is the job without partitions, which says so more
|
|
155
|
+
plainly by leaving the setting out. `count: auto` lets the job choose, as
|
|
156
|
+
it starts; see jobs.partitions.automaticCount. `partitions: auto` chooses
|
|
157
|
+
the column too, and is held as a PartitionsConfig with no column.
|
|
158
|
+
"""
|
|
159
|
+
|
|
160
|
+
model_config = ConfigDict(extra='forbid')
|
|
161
|
+
|
|
162
|
+
column: Optional[str] = Field(default=None, min_length=1)
|
|
163
|
+
count: Union[Literal['auto'], Annotated[int, Field(ge=2)]]
|
|
164
|
+
|
|
165
|
+
@model_validator(mode='before')
|
|
166
|
+
@classmethod
|
|
167
|
+
def _fromShorthand(cls, value: Any) -> Any:
|
|
168
|
+
|
|
169
|
+
if value == AUTOMATIC:
|
|
170
|
+
return {'count': AUTOMATIC, 'column': None}
|
|
171
|
+
if isinstance(value, Mapping) and value.get('column') is None:
|
|
172
|
+
raise ValueError('partitions names a column and a count, or is `auto` to choose both')
|
|
173
|
+
|
|
174
|
+
return value
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
@property
|
|
178
|
+
def automatic(self) -> bool:
|
|
179
|
+
"""Whether the job chooses how many slices, or the column too."""
|
|
180
|
+
|
|
181
|
+
return self.count == AUTOMATIC
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
AUTOMATIC = 'auto'
|
|
185
|
+
|
|
186
|
+
|
|
148
187
|
def _passesUnnamedColumns(masking: MaskingConfig) -> bool:
|
|
149
188
|
"""Whether a policy's defaultStrategy copies the columns it doesn't name
|
|
150
189
|
as they are: `keep`, or a custom strategy that says it passes values through.
|
|
@@ -203,6 +242,10 @@ class DataJobConfig(BaseJobConfig):
|
|
|
203
242
|
# the identifier fields of a table the job creates. An existing table's
|
|
204
243
|
# own identifier fields serve without it.
|
|
205
244
|
targetKey: CleanedStringList = Field(default_factory=list)
|
|
245
|
+
# A database target's alone: the job's rows read, masked and written as
|
|
246
|
+
# several slices at once. See PartitionsConfig. `auto` is ignored for a
|
|
247
|
+
# files or Iceberg target, so it can sit under `defaults`.
|
|
248
|
+
partitions: Optional[PartitionsConfig] = None
|
|
206
249
|
|
|
207
250
|
@field_validator('targetColumnTypes')
|
|
208
251
|
@classmethod
|
|
@@ -225,7 +268,8 @@ class DataJobConfig(BaseJobConfig):
|
|
|
225
268
|
"""
|
|
226
269
|
|
|
227
270
|
if self.insertStrategy in FILE_STRATEGIES:
|
|
228
|
-
tableOnly = [name for name in ('targetTableStage', 'preTargetAdhocQueries', 'postTargetAdhocQueries'
|
|
271
|
+
tableOnly = [name for name in ('targetTableStage', 'preTargetAdhocQueries', 'postTargetAdhocQueries', 'partitions')
|
|
272
|
+
if getattr(self, name) and not (name == 'partitions' and self.partitions is not None and self.partitions.column is None)]
|
|
229
273
|
if tableOnly:
|
|
230
274
|
raise ValueError('{} {} for a table in a database; insertStrategy: {} writes to files or Iceberg'.format(
|
|
231
275
|
_listed(tableOnly), 'is' if len(tableOnly) == 1 else 'are', self.insertStrategy.value))
|
|
@@ -389,6 +433,9 @@ _TARGET_SETTINGS = {
|
|
|
389
433
|
'targetTableStage': ('database',),
|
|
390
434
|
'preTargetAdhocQueries': ('database',),
|
|
391
435
|
'postTargetAdhocQueries': ('database',),
|
|
436
|
+
# A files or Iceberg target publishes what one writer wrote, and has no
|
|
437
|
+
# table for several to load into at once.
|
|
438
|
+
'partitions': ('database',),
|
|
392
439
|
}
|
|
393
440
|
|
|
394
441
|
_TARGET_NAMES = {'database': 'a database', 'files': 'a files connection', 'iceberg': 'an Iceberg connection'}
|
|
@@ -426,6 +473,9 @@ def targetProblems(job: DataJobConfig, connection: Any) -> List[str]:
|
|
|
426
473
|
what, _listed([strategy.value for strategy in _STRATEGIES[kind]]).replace(' and ', ' or '), job.insertStrategy.value))
|
|
427
474
|
|
|
428
475
|
for setting, kinds in _TARGET_SETTINGS.items():
|
|
476
|
+
if setting == 'partitions' and job.partitions is not None and job.partitions.column is None:
|
|
477
|
+
# `partitions: auto` reads a lake target's job as one stream.
|
|
478
|
+
continue
|
|
429
479
|
if getattr(job, setting) and kind not in kinds:
|
|
430
480
|
problems.append('{} is for {}, and it is {}'.format(
|
|
431
481
|
setting, ' or '.join(_TARGET_NAMES[other] for other in kinds), _TARGET_NAMES[kind]))
|
|
@@ -451,6 +501,31 @@ def targetProblems(job: DataJobConfig, connection: Any) -> List[str]:
|
|
|
451
501
|
return problems
|
|
452
502
|
|
|
453
503
|
|
|
504
|
+
def partitionLimitProblems(job: DataJobConfig, connections: Mapping[str, ConnectionConfig]) -> List[str]:
|
|
505
|
+
"""Why `job`'s partitions can't all run against its connections: each
|
|
506
|
+
partition opens a connection of its own to the source and the target, so
|
|
507
|
+
a partitioned job takes one of a connection's maxConcurrentJobs places for
|
|
508
|
+
each, and a job needing more places than there are could never start.
|
|
509
|
+
"""
|
|
510
|
+
|
|
511
|
+
if job.partitions is None or job.partitions.automatic:
|
|
512
|
+
# An automatic count is fitted to what the connections have free.
|
|
513
|
+
return []
|
|
514
|
+
|
|
515
|
+
problems = []
|
|
516
|
+
for setting in ('sourceConnection', 'targetConnection'):
|
|
517
|
+
alias = getattr(job, setting)
|
|
518
|
+
connection = connections.get(alias)
|
|
519
|
+
limit = None if connection is None else connection.jobLimit()
|
|
520
|
+
if limit is not None and int(job.partitions.count) > limit:
|
|
521
|
+
why = ('DuckDB lets one process at a time open a file' if isinstance(connection, DuckDBConnection)
|
|
522
|
+
else 'its maxConcurrentJobs is {}'.format(limit))
|
|
523
|
+
problems.append('partitions.count is {}, and {} "{}" takes at most {} at once ({}); each partition holds one of its places, '
|
|
524
|
+
'as a job does'.format(job.partitions.count, setting, alias, limit, why))
|
|
525
|
+
|
|
526
|
+
return problems
|
|
527
|
+
|
|
528
|
+
|
|
454
529
|
class NameRuleConfig(BaseModel):
|
|
455
530
|
"""A discovery.yaml rule on column names."""
|
|
456
531
|
|
|
@@ -554,7 +629,7 @@ StorageLocation = Union[Annotated[str, Field(min_length=1)], TableLocation]
|
|
|
554
629
|
# settings. A job that names any of these itself keeps its own value.
|
|
555
630
|
DEFAULTABLE_JOB_FIELDS = frozenset({
|
|
556
631
|
'active', 'refresh', 'sourceConnection', 'targetConnection', 'insertStrategy',
|
|
557
|
-
'chunkSize', 'retries', 'retryDelaySeconds', 'timeoutSeconds',
|
|
632
|
+
'chunkSize', 'retries', 'retryDelaySeconds', 'timeoutSeconds', 'partitions',
|
|
558
633
|
})
|
|
559
634
|
|
|
560
635
|
# Within `masking`, the key alone. A key reference is not a policy: `columns`
|
|
@@ -633,6 +708,10 @@ class DataJobsFile(BaseModel):
|
|
|
633
708
|
# as each job starts with the jobs running alongside it. See
|
|
634
709
|
# masking.maskingThreadsFor and runner._runCycle.
|
|
635
710
|
maskingThreads: Union[Literal['auto'], Annotated[int, Field(ge=1)]] = 1
|
|
711
|
+
# Whether a run stops before it starts, rather than masking in Python, when
|
|
712
|
+
# the native masker isn't installed or isn't this version. See
|
|
713
|
+
# masking.requireNativeProblem.
|
|
714
|
+
requireNative: bool = False
|
|
636
715
|
# Tables no job copies, on purpose: connection alias -> table -> why. What
|
|
637
716
|
# `bauta coverage` reads, so a table left out is a decision on the page
|
|
638
717
|
# rather than something nobody noticed.
|
|
@@ -720,7 +799,9 @@ class Configuration:
|
|
|
720
799
|
try:
|
|
721
800
|
return schema.model_validate(rawConfiguration)
|
|
722
801
|
except ValidationError as error:
|
|
723
|
-
|
|
802
|
+
# Not chained: pydantic's own error quotes the input it was given,
|
|
803
|
+
# a masking key or a password among it, in any traceback logged.
|
|
804
|
+
raise Configuration._invalid(error, sourceDescription) from None
|
|
724
805
|
|
|
725
806
|
|
|
726
807
|
@staticmethod
|
|
@@ -752,7 +833,7 @@ class Configuration:
|
|
|
752
833
|
try:
|
|
753
834
|
return _CONNECTION_ADAPTER.validate_python(rawConnection)
|
|
754
835
|
except ValidationError as error:
|
|
755
|
-
raise Configuration._invalid(error, sourceDescription, skipLocation=1) from
|
|
836
|
+
raise Configuration._invalid(error, sourceDescription, skipLocation=1) from None
|
|
756
837
|
|
|
757
838
|
|
|
758
839
|
@staticmethod
|
|
@@ -841,6 +922,7 @@ class Configuration:
|
|
|
841
922
|
jobName, job.sourceConnection, _TARGET_NAMES[targetKind(source)]))
|
|
842
923
|
for problem in ([] if target is None else targetProblems(job, target)):
|
|
843
924
|
problems.append('{}: targetConnection "{}" {}'.format(jobName, job.targetConnection, problem))
|
|
925
|
+
problems.extend('{}: {}'.format(jobName, problem) for problem in partitionLimitProblems(job, connections))
|
|
844
926
|
|
|
845
927
|
if connectionAliases is not None and isinstance(job, DataJobConfig):
|
|
846
928
|
if job.sourceConnection not in connectionAliases:
|
|
@@ -600,6 +600,57 @@ class Database:
|
|
|
600
600
|
self.alter(query=query)
|
|
601
601
|
|
|
602
602
|
|
|
603
|
+
def copyKeys(self, fromTable: str, toTable: str) -> List[str]:
|
|
604
|
+
"""Gives `toTable` the primary key and unique keys of `fromTable` that it
|
|
605
|
+
lacks, and says what it gave, as `primary key (id)` or `unique (email)`.
|
|
606
|
+
For a swap's stage table, which becomes the target, so the target keeps
|
|
607
|
+
its keys whichever table holds its name. See "How a swap works" in
|
|
608
|
+
docs/concepts/how-it-works.md.
|
|
609
|
+
|
|
610
|
+
Nothing is taken away, and a key `toTable` already has over the same
|
|
611
|
+
columns, in any order, is left as it is. Its columns are named as
|
|
612
|
+
`toTable` spells them, which must hold every one. Foreign keys are not
|
|
613
|
+
copied, for the reason `bauta schema` gives its stage tables none.
|
|
614
|
+
"""
|
|
615
|
+
|
|
616
|
+
def folded(columns: Sequence[str]) -> frozenset:
|
|
617
|
+
return frozenset(column.upper() for column in columns)
|
|
618
|
+
|
|
619
|
+
primaryKey = self.dialect.primaryKey(self.cursor, fromTable)
|
|
620
|
+
uniqueKeys = self.dialect.uniqueKeys(self.cursor, fromTable)
|
|
621
|
+
ownPrimaryKey = self.dialect.primaryKey(self.cursor, toTable)
|
|
622
|
+
held = {folded(columns) for columns in self.dialect.uniqueKeys(self.cursor, toTable)}
|
|
623
|
+
if ownPrimaryKey:
|
|
624
|
+
held.add(folded(ownPrimaryKey))
|
|
625
|
+
|
|
626
|
+
addPrimaryKey = primaryKey if primaryKey and not ownPrimaryKey else []
|
|
627
|
+
if addPrimaryKey:
|
|
628
|
+
held.add(folded(addPrimaryKey))
|
|
629
|
+
addUnique = []
|
|
630
|
+
for columns in uniqueKeys:
|
|
631
|
+
if folded(columns) not in held:
|
|
632
|
+
held.add(folded(columns))
|
|
633
|
+
addUnique.append(columns)
|
|
634
|
+
|
|
635
|
+
if not addPrimaryKey and not addUnique:
|
|
636
|
+
return []
|
|
637
|
+
|
|
638
|
+
spelled = [self.catalogColumns(table=toTable, columns=list(columns)) for columns in [addPrimaryKey] + addUnique]
|
|
639
|
+
try:
|
|
640
|
+
self.dialect.addKeys(self.cursor, self.statementName(toTable), toTable, self.quoted(spelled[0]),
|
|
641
|
+
[self.quoted(columns) for columns in spelled[1:]])
|
|
642
|
+
self.connection.commit()
|
|
643
|
+
except BaseException:
|
|
644
|
+
self.connection.rollback()
|
|
645
|
+
raise
|
|
646
|
+
finally:
|
|
647
|
+
self.primaryKeyCache.pop(toTable, None)
|
|
648
|
+
self.columnNameCache.pop(toTable, None)
|
|
649
|
+
|
|
650
|
+
return ((['primary key ({})'.format(', '.join(spelled[0]))] if addPrimaryKey else [])
|
|
651
|
+
+ ['unique ({})'.format(', '.join(columns)) for columns in spelled[1:]])
|
|
652
|
+
|
|
653
|
+
|
|
603
654
|
def swap(self, targetTable: str, stageTable: str) -> None:
|
|
604
655
|
"""Exchanges the two tables by renaming, atomically everywhere but
|
|
605
656
|
Oracle, through a temporary name in the stage table's schema.
|
|
@@ -5,6 +5,7 @@ from __future__ import annotations
|
|
|
5
5
|
|
|
6
6
|
import contextlib
|
|
7
7
|
import itertools
|
|
8
|
+
import re
|
|
8
9
|
from abc import ABC, abstractmethod
|
|
9
10
|
from enum import Enum
|
|
10
11
|
from typing import Any, Dict, List, NamedTuple, Optional, Sequence, Tuple, Type, TypeVar
|
|
@@ -69,6 +70,54 @@ def _groupForeignKeys(rows: Sequence[Sequence[Any]]) -> List[ForeignKey]:
|
|
|
69
70
|
]
|
|
70
71
|
|
|
71
72
|
|
|
73
|
+
def _uniqueColumnGroups(rows: Sequence[Sequence[Any]]) -> List[Tuple[str, ...]]:
|
|
74
|
+
"""Folds (keyName, column) rows, ordered by position within each key, into
|
|
75
|
+
each key's columns, leaving out a key with a NULL column -- an
|
|
76
|
+
expression, or a prefix of a column -- and a column list already seen.
|
|
77
|
+
"""
|
|
78
|
+
|
|
79
|
+
grouped: Dict[str, List[Optional[str]]] = {}
|
|
80
|
+
for name, column in rows:
|
|
81
|
+
grouped.setdefault(name, []).append(column)
|
|
82
|
+
|
|
83
|
+
groups: List[Tuple[str, ...]] = []
|
|
84
|
+
for columns in grouped.values():
|
|
85
|
+
if any(column is None for column in columns):
|
|
86
|
+
continue
|
|
87
|
+
group = tuple(str(column) for column in columns)
|
|
88
|
+
if group not in groups:
|
|
89
|
+
groups.append(group)
|
|
90
|
+
|
|
91
|
+
return groups
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
_TABLE_NAME_PART = r'(?:"(?:[^"]|"")*"|\[[^\]]*\]|`(?:[^`]|``)*`|[^\s.(]+)'
|
|
95
|
+
|
|
96
|
+
# The head of a CREATE TABLE statement as SQLite and DuckDB keep it, up to the
|
|
97
|
+
# end of the table's name, schema-qualified or not.
|
|
98
|
+
_CREATE_TABLE_HEAD = re.compile(r'^\s*CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?{0}(?:\s*\.\s*{0})?'.format(_TABLE_NAME_PART), re.IGNORECASE)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _withKeys(createStatement: str, table: str, primaryKey: Sequence[str], uniqueKeys: Sequence[Sequence[str]]) -> str:
|
|
102
|
+
"""A table's own CREATE TABLE statement, naming it as `table` spells it,
|
|
103
|
+
with a primary key and unique constraints added after its last column.
|
|
104
|
+
|
|
105
|
+
The column list's closing parenthesis is the statement's last: what may
|
|
106
|
+
follow it -- WITHOUT ROWID, STRICT -- are words.
|
|
107
|
+
"""
|
|
108
|
+
|
|
109
|
+
head = _CREATE_TABLE_HEAD.match(createStatement)
|
|
110
|
+
if head is None:
|
|
111
|
+
raise ConfigurationError('could not read the statement that created {}'.format(table))
|
|
112
|
+
|
|
113
|
+
statement = 'CREATE TABLE {}{}'.format(table, createStatement[head.end():]).rstrip().rstrip(';').rstrip()
|
|
114
|
+
closing = statement.rindex(')')
|
|
115
|
+
clauses = (['PRIMARY KEY ({})'.format(', '.join(primaryKey))] if primaryKey else []) + \
|
|
116
|
+
['UNIQUE ({})'.format(', '.join(columns)) for columns in uniqueKeys]
|
|
117
|
+
|
|
118
|
+
return '{}, {}{}'.format(statement[:closing], ', '.join(clauses), statement[closing:])
|
|
119
|
+
|
|
120
|
+
|
|
72
121
|
class ColumnDefinition(NamedTuple):
|
|
73
122
|
"""One column as the database's catalog describes it, for generating DDL.
|
|
74
123
|
|
|
@@ -335,6 +384,39 @@ class DatabaseDialect(ABC):
|
|
|
335
384
|
|
|
336
385
|
return [row[0] for row in self._catalog(cursor, self.primaryKeyQuery(), table)]
|
|
337
386
|
|
|
387
|
+
def uniqueKeysQuery(self) -> str:
|
|
388
|
+
"""One table's unique constraints and unique indexes, its primary key
|
|
389
|
+
aside, as rows of (name, column) ordered by name and position. A
|
|
390
|
+
column that is an expression, or only a prefix of one, comes back as
|
|
391
|
+
NULL, and its key is left out: copying it would take the database's
|
|
392
|
+
own DDL, not a column list.
|
|
393
|
+
"""
|
|
394
|
+
|
|
395
|
+
raise NotImplementedError('{} cannot describe unique keys'.format(type(self).__name__))
|
|
396
|
+
|
|
397
|
+
def uniqueKeys(self, cursor: Cursor, table: str) -> List[Tuple[str, ...]]:
|
|
398
|
+
"""The column lists of the table's unique keys over plain columns, each
|
|
399
|
+
once, its primary key aside. A partial or filtered one is left out by
|
|
400
|
+
each dialect's query, since it isn't unique over every row.
|
|
401
|
+
"""
|
|
402
|
+
|
|
403
|
+
return _uniqueColumnGroups(self._catalog(cursor, self.uniqueKeysQuery(), table))
|
|
404
|
+
|
|
405
|
+
def addKeys(self, cursor: Cursor, table: str, catalogTable: str, primaryKey: Sequence[str], uniqueKeys: Sequence[Sequence[str]]) -> None:
|
|
406
|
+
"""Gives `table`, as a statement names it, a primary key and unique
|
|
407
|
+
constraints over the quoted columns given; the caller commits.
|
|
408
|
+
|
|
409
|
+
Unnamed, so each database names them as it would any such constraint,
|
|
410
|
+
and no name can collide with one the swap's other table already has
|
|
411
|
+
-- the target's own, which keep their names when the tables trade
|
|
412
|
+
theirs. PostgreSQL, MySQL, MariaDB and Oracle take this as it is.
|
|
413
|
+
"""
|
|
414
|
+
|
|
415
|
+
if primaryKey:
|
|
416
|
+
cursor.execute('ALTER TABLE {} ADD PRIMARY KEY ({})'.format(table, ', '.join(primaryKey)))
|
|
417
|
+
for columns in uniqueKeys:
|
|
418
|
+
cursor.execute('ALTER TABLE {} ADD UNIQUE ({})'.format(table, ', '.join(columns)))
|
|
419
|
+
|
|
338
420
|
def columnDefinitions(self, cursor: Cursor, table: str) -> List[ColumnDefinition]:
|
|
339
421
|
|
|
340
422
|
return _columnDefinitions(self._catalog(cursor, self.columnsQuery(), table))
|
|
@@ -10,7 +10,7 @@ from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, cast
|
|
|
10
10
|
|
|
11
11
|
from ..driver import Connection, Cursor
|
|
12
12
|
from ...configuration import ConfigurationError, DatabaseConfig, DatabaseType, DuckDBConnection
|
|
13
|
-
from .base import ColumnCategory, ForeignKey, settingsOf, _OnConflictDialect, _renameInThreeSteps, _schemaForeignKeys
|
|
13
|
+
from .base import ColumnCategory, ForeignKey, settingsOf, _OnConflictDialect, _renameInThreeSteps, _schemaForeignKeys, _uniqueColumnGroups, _withKeys
|
|
14
14
|
from .names import catalogTableName
|
|
15
15
|
|
|
16
16
|
|
|
@@ -344,6 +344,46 @@ class DuckDBDialect(_OnConflictDialect):
|
|
|
344
344
|
"AND lower(table_name) = lower({}) AND database_name = current_database()")
|
|
345
345
|
|
|
346
346
|
|
|
347
|
+
def uniqueKeys(self, cursor: Cursor, table: str) -> List[Tuple[str, ...]]:
|
|
348
|
+
"""UNIQUE constraints, which DuckDB lists apart from its primary key,
|
|
349
|
+
each with its columns as an array. A CREATE UNIQUE INDEX is no key a
|
|
350
|
+
swap could keep: DuckDB won't rename a table with an index.
|
|
351
|
+
"""
|
|
352
|
+
|
|
353
|
+
schema, name = catalogTableName(self.databaseType, table)
|
|
354
|
+
cursor.execute("SELECT constraint_column_names FROM duckdb_constraints() WHERE constraint_type = 'UNIQUE' "
|
|
355
|
+
"AND lower(schema_name) = lower(COALESCE(?, current_schema())) AND lower(table_name) = lower(?) "
|
|
356
|
+
"AND database_name = current_database() ORDER BY constraint_index", (schema, name))
|
|
357
|
+
|
|
358
|
+
return _uniqueColumnGroups([(str(index), column) for index, (columns,) in enumerate(cursor.fetchall()) for column in columns])
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def addKeys(self, cursor: Cursor, table: str, catalogTable: str, primaryKey: Sequence[str], uniqueKeys: Sequence[Sequence[str]]) -> None:
|
|
362
|
+
"""DuckDB adds no UNIQUE constraint to a table, so the table is created
|
|
363
|
+
again from its own statement with the keys added, in the session's one
|
|
364
|
+
transaction. Only an empty table: a stage table just emptied for its
|
|
365
|
+
load.
|
|
366
|
+
"""
|
|
367
|
+
|
|
368
|
+
schema, name = catalogTableName(self.databaseType, catalogTable)
|
|
369
|
+
where = ("lower(schema_name) = lower(COALESCE(?, current_schema())) AND lower(table_name) = lower(?) "
|
|
370
|
+
"AND database_name = current_database()")
|
|
371
|
+
|
|
372
|
+
cursor.execute('SELECT 1 FROM {} LIMIT 1'.format(table))
|
|
373
|
+
if cursor.fetchone() is not None:
|
|
374
|
+
raise ConfigurationError('{} holds rows, and DuckDB can only add a key by creating the table again'.format(catalogTable))
|
|
375
|
+
|
|
376
|
+
cursor.execute('SELECT sql FROM duckdb_tables() WHERE {}'.format(where), (schema, name))
|
|
377
|
+
created = cursor.fetchone()[0]
|
|
378
|
+
cursor.execute('SELECT sql FROM duckdb_indexes() WHERE {} AND sql IS NOT NULL'.format(where), (schema, name))
|
|
379
|
+
indexes = [row[0] for row in cursor.fetchall()]
|
|
380
|
+
|
|
381
|
+
cursor.execute('DROP TABLE {}'.format(table))
|
|
382
|
+
cursor.execute(_withKeys(created, table, primaryKey, uniqueKeys))
|
|
383
|
+
for index in indexes:
|
|
384
|
+
cursor.execute(index)
|
|
385
|
+
|
|
386
|
+
|
|
347
387
|
def tableExistsQuery(self) -> str:
|
|
348
388
|
|
|
349
389
|
return ("SELECT count(*) FROM information_schema.tables "
|