bauta 0.2.3__tar.gz → 0.2.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. {bauta-0.2.3 → bauta-0.2.4}/PKG-INFO +3 -3
  2. {bauta-0.2.3 → bauta-0.2.4}/bauta/cli/__init__.py +15 -2
  3. {bauta-0.2.3 → bauta-0.2.4}/bauta/cli/common.py +19 -0
  4. {bauta-0.2.3 → bauta-0.2.4}/bauta/cli/review.py +27 -6
  5. {bauta-0.2.3 → bauta-0.2.4}/bauta/cli/run.py +12 -1
  6. {bauta-0.2.3 → bauta-0.2.4}/bauta/configuration/__init__.py +4 -2
  7. {bauta-0.2.3 → bauta-0.2.4}/bauta/configuration/models.py +87 -5
  8. {bauta-0.2.3 → bauta-0.2.4}/bauta/database/connection.py +51 -0
  9. {bauta-0.2.3 → bauta-0.2.4}/bauta/database/dialects/base.py +82 -0
  10. {bauta-0.2.3 → bauta-0.2.4}/bauta/database/dialects/duckdb.py +41 -1
  11. {bauta-0.2.3 → bauta-0.2.4}/bauta/database/dialects/mssql.py +55 -0
  12. {bauta-0.2.3 → bauta-0.2.4}/bauta/database/dialects/mysql.py +11 -0
  13. {bauta-0.2.3 → bauta-0.2.4}/bauta/database/dialects/oracle.py +14 -0
  14. {bauta-0.2.3 → bauta-0.2.4}/bauta/database/dialects/postgresql.py +18 -0
  15. {bauta-0.2.3 → bauta-0.2.4}/bauta/database/dialects/sqlite.py +60 -3
  16. {bauta-0.2.3 → bauta-0.2.4}/bauta/generate/builtinDiscovery.py +9 -4
  17. {bauta-0.2.3 → bauta-0.2.4}/bauta/generate/discovery.py +70 -9
  18. {bauta-0.2.3 → bauta-0.2.4}/bauta/generate/synthesize.py +4 -2
  19. {bauta-0.2.3 → bauta-0.2.4}/bauta/jobs/__init__.py +2 -1
  20. {bauta-0.2.3 → bauta-0.2.4}/bauta/jobs/dependencyGraph.py +46 -2
  21. {bauta-0.2.3 → bauta-0.2.4}/bauta/jobs/keys.py +39 -18
  22. bauta-0.2.4/bauta/jobs/partitions.py +193 -0
  23. {bauta-0.2.3 → bauta-0.2.4}/bauta/jobs/pipeline.py +292 -36
  24. {bauta-0.2.3 → bauta-0.2.4}/bauta/jobs/runner.py +28 -2
  25. {bauta-0.2.3 → bauta-0.2.4}/bauta/jobs/targets.py +70 -5
  26. {bauta-0.2.3 → bauta-0.2.4}/bauta/jobs/workers.py +8 -4
  27. {bauta-0.2.3 → bauta-0.2.4}/bauta/lake/iceberg.py +2 -1
  28. {bauta-0.2.3 → bauta-0.2.4}/bauta/lake/target.py +2 -1
  29. {bauta-0.2.3 → bauta-0.2.4}/bauta/log/__init__.py +4 -1
  30. {bauta-0.2.3 → bauta-0.2.4}/bauta/log/core.py +58 -4
  31. bauta-0.2.4/bauta/log/scrubbing.py +258 -0
  32. {bauta-0.2.3 → bauta-0.2.4}/bauta/masking/__init__.py +11 -4
  33. {bauta-0.2.3 → bauta-0.2.4}/bauta/masking/core.py +225 -14
  34. bauta-0.2.4/bauta/masking/fakeData.py +563 -0
  35. {bauta-0.2.3 → bauta-0.2.4}/bauta/masking/strategies.py +390 -27
  36. {bauta-0.2.3 → bauta-0.2.4}/bauta/review/audit.py +147 -18
  37. bauta-0.2.4/bauta/review/html.py +259 -0
  38. {bauta-0.2.3 → bauta-0.2.4}/bauta.egg-info/PKG-INFO +3 -3
  39. {bauta-0.2.3 → bauta-0.2.4}/bauta.egg-info/SOURCES.txt +2 -0
  40. {bauta-0.2.3 → bauta-0.2.4}/bauta.egg-info/requires.txt +2 -2
  41. {bauta-0.2.3 → bauta-0.2.4}/pyproject.toml +7 -4
  42. bauta-0.2.3/bauta/log/scrubbing.py +0 -115
  43. bauta-0.2.3/bauta/masking/fakeData.py +0 -162
  44. {bauta-0.2.3 → bauta-0.2.4}/LICENSE +0 -0
  45. {bauta-0.2.3 → bauta-0.2.4}/README.md +0 -0
  46. {bauta-0.2.3 → bauta-0.2.4}/bauta/__init__.py +0 -0
  47. {bauta-0.2.3 → bauta-0.2.4}/bauta/__main__.py +0 -0
  48. {bauta-0.2.3 → bauta-0.2.4}/bauta/cli/generate.py +0 -0
  49. {bauta-0.2.3 → bauta-0.2.4}/bauta/configuration/columnTypes.py +0 -0
  50. {bauta-0.2.3 → bauta-0.2.4}/bauta/configuration/connections.py +0 -0
  51. {bauta-0.2.3 → bauta-0.2.4}/bauta/configuration/environment.py +0 -0
  52. {bauta-0.2.3 → bauta-0.2.4}/bauta/configuration/files.py +0 -0
  53. {bauta-0.2.3 → bauta-0.2.4}/bauta/database/__init__.py +0 -0
  54. {bauta-0.2.3 → bauta-0.2.4}/bauta/database/dialects/__init__.py +0 -0
  55. {bauta-0.2.3 → bauta-0.2.4}/bauta/database/dialects/names.py +0 -0
  56. {bauta-0.2.3 → bauta-0.2.4}/bauta/database/driver.py +0 -0
  57. {bauta-0.2.3 → bauta-0.2.4}/bauta/database/values.py +0 -0
  58. {bauta-0.2.3 → bauta-0.2.4}/bauta/generate/__init__.py +0 -0
  59. {bauta-0.2.3 → bauta-0.2.4}/bauta/generate/drift.py +0 -0
  60. {bauta-0.2.3 → bauta-0.2.4}/bauta/generate/schema.py +0 -0
  61. {bauta-0.2.3 → bauta-0.2.4}/bauta/generate/subset.py +0 -0
  62. {bauta-0.2.3 → bauta-0.2.4}/bauta/jobs/fullRefresh.py +0 -0
  63. {bauta-0.2.3 → bauta-0.2.4}/bauta/jobs/memory.py +0 -0
  64. {bauta-0.2.3 → bauta-0.2.4}/bauta/jobs/reporting.py +0 -0
  65. {bauta-0.2.3 → bauta-0.2.4}/bauta/lake/__init__.py +0 -0
  66. {bauta-0.2.3 → bauta-0.2.4}/bauta/lake/columnar.py +0 -0
  67. {bauta-0.2.3 → bauta-0.2.4}/bauta/lake/columns.py +0 -0
  68. {bauta-0.2.3 → bauta-0.2.4}/bauta/lake/formats.py +0 -0
  69. {bauta-0.2.3 → bauta-0.2.4}/bauta/lake/stores.py +0 -0
  70. {bauta-0.2.3 → bauta-0.2.4}/bauta/masking/fpe.py +0 -0
  71. {bauta-0.2.3 → bauta-0.2.4}/bauta/py.typed +0 -0
  72. {bauta-0.2.3 → bauta-0.2.4}/bauta/review/__init__.py +0 -0
  73. {bauta-0.2.3 → bauta-0.2.4}/bauta/review/coverage.py +0 -0
  74. {bauta-0.2.3 → bauta-0.2.4}/bauta/review/references.py +0 -0
  75. {bauta-0.2.3 → bauta-0.2.4}/bauta/transform/__init__.py +0 -0
  76. {bauta-0.2.3 → bauta-0.2.4}/bauta/transform/builtinTransforms.py +0 -0
  77. {bauta-0.2.3 → bauta-0.2.4}/bauta/transform/core.py +0 -0
  78. {bauta-0.2.3 → bauta-0.2.4}/bauta.egg-info/dependency_links.txt +0 -0
  79. {bauta-0.2.3 → bauta-0.2.4}/bauta.egg-info/entry_points.txt +0 -0
  80. {bauta-0.2.3 → bauta-0.2.4}/bauta.egg-info/top_level.txt +0 -0
  81. {bauta-0.2.3 → bauta-0.2.4}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: bauta
3
- Version: 0.2.3
3
+ Version: 0.2.4
4
4
  Summary: Safe, realistic copies of production across databases: masking, subsets, synthetic data, and the ETL to move them
5
5
  Author: David Ribeiro
6
6
  License-Expression: MIT
@@ -42,7 +42,7 @@ Provides-Extra: mariadb
42
42
  Requires-Dist: mysql-connector-python>=8.4; extra == "mariadb"
43
43
  Provides-Extra: sqlite
44
44
  Provides-Extra: duckdb
45
- Requires-Dist: duckdb>=1.4; extra == "duckdb"
45
+ Requires-Dist: duckdb>=1.5; extra == "duckdb"
46
46
  Requires-Dist: pyarrow>=21; extra == "duckdb"
47
47
  Requires-Dist: pytz>=2024.1; extra == "duckdb"
48
48
  Provides-Extra: files
@@ -55,7 +55,7 @@ Requires-Dist: cryptography>=42; extra == "fpe"
55
55
  Provides-Extra: all
56
56
  Requires-Dist: bauta[duckdb,files,fpe,iceberg,mariadb,mssql,mysql,oracle,postgresql,sqlite]; extra == "all"
57
57
  Provides-Extra: native
58
- Requires-Dist: bauta-rs==0.2.3; extra == "native"
58
+ Requires-Dist: bauta-rs==0.2.4; extra == "native"
59
59
  Provides-Extra: docs
60
60
  Requires-Dist: mkdocs==1.6.1; extra == "docs"
61
61
  Requires-Dist: mkdocs-material==9.7.7; extra == "docs"
@@ -89,6 +89,15 @@ def _positiveInteger(text: str) -> int:
89
89
  return value
90
90
 
91
91
 
92
+ def _nonNegativeInteger(text: str) -> int:
93
+
94
+ value = int(text)
95
+ if value < 0:
96
+ raise argparse.ArgumentTypeError('must be at least 0, got {}'.format(value))
97
+
98
+ return value
99
+
100
+
92
101
  def _addRunArguments(parser: argparse.ArgumentParser) -> None:
93
102
 
94
103
  parser.add_argument('--forever', action='store_true',
@@ -179,7 +188,10 @@ def _buildParser() -> argparse.ArgumentParser:
179
188
  auditParser.add_argument('--connect', action='store_true',
180
189
  help='also run each masked query for its real columns, and check whether each connection is encrypted')
181
190
  auditParser.add_argument('--job', action='append', help='audit only this job (repeatable)')
182
- auditParser.add_argument('--format', default='text', choices=['text', 'json'], help='default: text')
191
+ auditParser.add_argument('--sample', type=_nonNegativeInteger, default=1000, metavar='ROWS',
192
+ help='with --connect, rows of each query read to check kept columns and shared domains by their values; 0 reads none')
193
+ auditParser.add_argument('--format', default='text', choices=['text', 'json', 'html'],
194
+ help='text, json, or html: one self-contained page for a reviewer, best with --output (default: text)')
183
195
  auditParser.add_argument('--strict', action='store_true', help='exit 1 on warnings as well as errors')
184
196
  auditParser.add_argument('--output', help='write the report here instead of stdout; must not already exist')
185
197
  _addRulesArgument(auditParser)
@@ -198,7 +210,8 @@ def _buildParser() -> argparse.ArgumentParser:
198
210
  coverageParser.add_argument('--connection', help='the source connection alias to check; required when the jobs read from more than one')
199
211
  coverageParser.add_argument('--schema', help='the schema to list, instead of the connection\'s own')
200
212
  coverageParser.add_argument('--job', action='append', help='only these jobs count as covering a table. Repeatable.')
201
- coverageParser.add_argument('--format', choices=['text', 'json'], default='text', help='output format')
213
+ coverageParser.add_argument('--format', choices=['text', 'json', 'html'], default='text',
214
+ help='text, json, or html: one self-contained page for a reviewer, best with --output (default: text)')
202
215
  coverageParser.add_argument('--output', help='write to this file instead of stdout')
203
216
 
204
217
  verifyParser = subparsers.add_parser('verify-manifest', help='check that a manifest is unaltered, and who signed it')
@@ -300,6 +300,25 @@ def _sourceQueryColumns(job: DataJobConfig, connections: _Connections) -> List[s
300
300
  return columns
301
301
 
302
302
 
303
+ def _sourceQuerySample(job: DataJobConfig, connections: _Connections, rows: int) -> Tuple[List[str], List[Tuple[Any, ...]]]:
304
+ """The columns a job's sourceQuery returns, and up to `rows` of its rows
305
+ from the start, without reading the rest. They stay in memory: nothing
306
+ that reads them may report a value.
307
+ """
308
+
309
+ with connections.use(job.sourceConnection) as database:
310
+ query = job.sourceQuery
311
+ parameters = None
312
+ if job.watermarkColumn:
313
+ query = database.substituteWatermarkPlaceholder(query)
314
+ parameters = (job.watermarkInitial,)
315
+ columns, chunks = database.stream(query=query, chunkSize=max(1, rows), parameters=parameters)
316
+ with chunks:
317
+ sampled = next(chunks, []) if rows else []
318
+
319
+ return columns, list(sampled)
320
+
321
+
303
322
  def _targetColumns(job: DataJobConfig, connections: _Connections) -> List[str]:
304
323
  """The target's columns, in the order a load fills them."""
305
324
 
@@ -13,7 +13,7 @@ from ..database.dialects import ForeignKey, bareName, splitTableName, unqualifie
13
13
  from ..log import Log
14
14
  from ..log.scrubbing import describeError
15
15
  from .common import (EXIT_JOBS_DID_NOT_SUCCEED, EXIT_SUCCESS, _Connections, UsageError, _discoveryRules, _loadDataJobs, _requireDatabase, _selectJobs,
16
- _sourceQueryColumns, _targetColumns, _writeOutput)
16
+ _sourceQuerySample, _targetColumns, _toolVersion, _writeOutput)
17
17
 
18
18
 
19
19
  def _commandVerifyReferences(arguments: argparse.Namespace, log: Log) -> int:
@@ -136,8 +136,16 @@ def _commandCoverage(arguments: argparse.Namespace, log: Log) -> int:
136
136
 
137
137
  report = coverageReport(alias, tables, jobs, acknowledged=jobsFile.acknowledged.get(alias),
138
138
  columns=columns, rules=_discoveryRules(arguments))
139
- _writeOutput(json.dumps(report, indent=2, default=str) + '\n' if arguments.format == 'json' else renderCoverage(report),
140
- arguments.output)
139
+ if arguments.format == 'html':
140
+ import datetime
141
+
142
+ from ..review.html import renderCoverageHtml
143
+
144
+ generatedAt = datetime.datetime.now(datetime.timezone.utc).isoformat(timespec='seconds')
145
+ text = renderCoverageHtml(report, generatedAt=generatedAt, version=_toolVersion())
146
+ else:
147
+ text = json.dumps(report, indent=2, default=str) + '\n' if arguments.format == 'json' else renderCoverage(report)
148
+ _writeOutput(text, arguments.output)
141
149
 
142
150
  if report['summary'][UNCOVERED]:
143
151
  return EXIT_JOBS_DID_NOT_SUCCEED
@@ -162,6 +170,11 @@ def _commandAudit(arguments: argparse.Namespace, log: Log) -> int:
162
170
  question. Offline unless --connect, which also resolves each masked
163
171
  query's real columns and checks whether each connection is encrypted.
164
172
 
173
+ With --connect it also reads up to --sample rows of each query, to
174
+ question a column kept as it is whose values look like personal data, and
175
+ columns sharing a domain that hold its values differently. The rows stay
176
+ in memory, and no value is reported.
177
+
165
178
  Exits 1 on an error finding, and with --strict on a warning too.
166
179
  """
167
180
 
@@ -177,6 +190,7 @@ def _commandAudit(arguments: argparse.Namespace, log: Log) -> int:
177
190
  foreignKeys: Dict[str, List[ForeignKey]] = {}
178
191
  declaredForeignKeys: Dict[str, List[ForeignKey]] = {}
179
192
  elsewhere: Dict[str, Dict[str, int]] = {}
193
+ samples: Dict[str, List[Tuple[Any, ...]]] = {}
180
194
 
181
195
  if arguments.connect:
182
196
  # Every job's columns, not only a masked one's: an unmasked job's are
@@ -186,7 +200,7 @@ def _commandAudit(arguments: argparse.Namespace, log: Log) -> int:
186
200
  with _Connections(connectionConfiguration) as connections:
187
201
  for name, job in jobs.items():
188
202
  try:
189
- returnedColumns[name] = _sourceQueryColumns(job, connections)
203
+ returnedColumns[name], samples[name] = _sourceQuerySample(job, connections, arguments.sample)
190
204
  except Exception as error:
191
205
  unreachable[name] = describeError(error)
192
206
  continue
@@ -242,8 +256,15 @@ def _commandAudit(arguments: argparse.Namespace, log: Log) -> int:
242
256
  report = auditJobs(jobs, returnedColumns=returnedColumns, encryption=encryption, unreachable=unreachable,
243
257
  targetColumns=targetColumns, foreignKeys=foreignKeys, declaredForeignKeys=declaredForeignKeys,
244
258
  rules=_discoveryRules(arguments), connections=connectionConfiguration,
245
- foreignKeysElsewhere=elsewhere if arguments.connect else None, unreadableTargets=unreadableTargets)
246
- _writeOutput(json.dumps(report, indent=2, default=str) + '\n' if arguments.format == 'json' else renderAudit(report), arguments.output)
259
+ foreignKeysElsewhere=elsewhere if arguments.connect else None, unreadableTargets=unreadableTargets,
260
+ samples=samples)
261
+ if arguments.format == 'html':
262
+ from ..review.html import renderAuditHtml
263
+
264
+ text = renderAuditHtml(report, version=_toolVersion())
265
+ else:
266
+ text = json.dumps(report, indent=2, default=str) + '\n' if arguments.format == 'json' else renderAudit(report)
267
+ _writeOutput(text, arguments.output)
247
268
 
248
269
  if report['summary']['error'] or (arguments.strict and report['summary']['warning']):
249
270
  return EXIT_JOBS_DID_NOT_SUCCEED
@@ -276,12 +276,16 @@ def _commandValidate(arguments: argparse.Namespace, log: Log) -> int:
276
276
  except ConfigurationError as error:
277
277
  problems.append('{}: {}'.format(alias, error))
278
278
 
279
- from ..masking import effectiveMaskingThreads
279
+ from ..masking import effectiveMaskingThreads, requireNativeProblem
280
280
 
281
281
  try:
282
282
  effectiveMaskingThreads(jobsFile.maskingThreads)
283
283
  except ValueError as error:
284
284
  problems.append(str(error))
285
+ if any(job.masking is not None and job.active for job in jobsFile.jobs.values()):
286
+ problem = requireNativeProblem(jobsFile.requireNative)
287
+ if problem:
288
+ problems.append(problem)
285
289
 
286
290
  if problems:
287
291
  raise ConfigurationError('invalid configuration:\n' + '\n'.join(problems))
@@ -414,6 +418,13 @@ def _dryRunDataJobs(jobsFile: DataJobsFile, connectionConfiguration: Dict[str, C
414
418
  problem = _checkColumnCounts(name, job, returned, returned if columns is None else columns)
415
419
  if problem is None and job.masking is not None:
416
420
  problem = _checkMaskingCoverage(name, job, returned, log)
421
+ if problem is None and job.partitions is not None and job.partitions.column is not None:
422
+ from ..jobs.partitions import resolveColumn
423
+
424
+ try:
425
+ resolveColumn(job.partitions.column, returned)
426
+ except ConfigurationError as error:
427
+ problem = '{}: {}'.format(name, error)
417
428
  if problem:
418
429
  problems.append(problem)
419
430
 
@@ -8,8 +8,8 @@ from .connections import (EMBEDDED_TYPES, IDENTIFIER, S3_LARGEST_COPY, Connectio
8
8
  FileCompression, FileFormat, FilesConnection, FileStore, IcebergCatalog, IcebergConnection, LakeConfig, MariaDBConnection, anchorPaths, isSchemePath,
9
9
  MSSQLConnection, MySQLConnection, OracleConnection, PostgreSQLConnection, SQLiteConnection, StoreType)
10
10
  from .models import (FILE_STRATEGIES, WATERMARK_PLACEHOLDER, BaseJobConfig, Configuration, DataJobConfig, DataJobsFile, DiscoveryRulesFile,
11
- InsertStrategy, MaskingConfig, NameRuleConfig, StorageLocation, TableLocation, ValueRuleConfig, connectionConfig, filePathProblem,
12
- findCycle, isLake, targetKind, targetProblems)
11
+ InsertStrategy, MaskingConfig, NameRuleConfig, PartitionsConfig, StorageLocation, TableLocation, ValueRuleConfig, connectionConfig, filePathProblem,
12
+ findCycle, isLake, partitionLimitProblems, targetKind, targetProblems)
13
13
  from .columnTypes import ColumnType, parseColumnType
14
14
  from .files import INCLUDE_KEY, JobsDocument, loadYamlFile, readJobsFile
15
15
 
@@ -53,6 +53,8 @@ __all__ = [
53
53
  'MySQLConnection',
54
54
  'NameRuleConfig',
55
55
  'OracleConnection',
56
+ 'partitionLimitProblems',
57
+ 'PartitionsConfig',
56
58
  'parseColumnType',
57
59
  'PasswordCommandError',
58
60
  'PostgreSQLConnection',
@@ -11,7 +11,7 @@ from typing import Annotated, Any, Callable, Dict, List, Literal, Mapping, Optio
11
11
 
12
12
  from pydantic import BaseModel, ConfigDict, Field, SecretStr, ValidationError, field_validator, model_validator
13
13
 
14
- from ..masking import changesValues, policyFor, validateColumnPolicy, validateKey
14
+ from ..masking import changesValues, policyFor, validateColumnPolicy, validateKey, warnIfWeakKey
15
15
  from .connections import (_CONNECTION_ADAPTER, CONNECTION_TYPES, anchorPaths, CleanedListMapping, CleanedMapping, CleanedStringList, ConnectionConfig,
16
16
  DuckDBConnection, FilesConnection, IcebergConnection, _listed)
17
17
  from .environment import ConfigurationError
@@ -127,6 +127,7 @@ class MaskingConfig(BaseModel):
127
127
  def _requireStrongKey(cls, key: SecretStr) -> SecretStr:
128
128
 
129
129
  validateKey(key.get_secret_value())
130
+ warnIfWeakKey(key.get_secret_value())
130
131
 
131
132
  return key
132
133
 
@@ -145,6 +146,44 @@ class MaskingConfig(BaseModel):
145
146
  return None if policy is None else validateColumnPolicy(policy)
146
147
 
147
148
 
149
+ class PartitionsConfig(BaseModel):
150
+ """A job read, masked and written as `count` slices at once, each a range
151
+ of the numeric `column` its sourceQuery returns. See "Partitions" in
152
+ docs/concepts/how-it-works.md.
153
+
154
+ At least two: one slice is the job without partitions, which says so more
155
+ plainly by leaving the setting out. `count: auto` lets the job choose, as
156
+ it starts; see jobs.partitions.automaticCount. `partitions: auto` chooses
157
+ the column too, and is held as a PartitionsConfig with no column.
158
+ """
159
+
160
+ model_config = ConfigDict(extra='forbid')
161
+
162
+ column: Optional[str] = Field(default=None, min_length=1)
163
+ count: Union[Literal['auto'], Annotated[int, Field(ge=2)]]
164
+
165
+ @model_validator(mode='before')
166
+ @classmethod
167
+ def _fromShorthand(cls, value: Any) -> Any:
168
+
169
+ if value == AUTOMATIC:
170
+ return {'count': AUTOMATIC, 'column': None}
171
+ if isinstance(value, Mapping) and value.get('column') is None:
172
+ raise ValueError('partitions names a column and a count, or is `auto` to choose both')
173
+
174
+ return value
175
+
176
+
177
+ @property
178
+ def automatic(self) -> bool:
179
+ """Whether the job chooses how many slices, or the column too."""
180
+
181
+ return self.count == AUTOMATIC
182
+
183
+
184
+ AUTOMATIC = 'auto'
185
+
186
+
148
187
  def _passesUnnamedColumns(masking: MaskingConfig) -> bool:
149
188
  """Whether a policy's defaultStrategy copies the columns it doesn't name
150
189
  as they are: `keep`, or a custom strategy that says it passes values through.
@@ -203,6 +242,10 @@ class DataJobConfig(BaseJobConfig):
203
242
  # the identifier fields of a table the job creates. An existing table's
204
243
  # own identifier fields serve without it.
205
244
  targetKey: CleanedStringList = Field(default_factory=list)
245
+ # A database target's alone: the job's rows read, masked and written as
246
+ # several slices at once. See PartitionsConfig. `auto` is ignored for a
247
+ # files or Iceberg target, so it can sit under `defaults`.
248
+ partitions: Optional[PartitionsConfig] = None
206
249
 
207
250
  @field_validator('targetColumnTypes')
208
251
  @classmethod
@@ -225,7 +268,8 @@ class DataJobConfig(BaseJobConfig):
225
268
  """
226
269
 
227
270
  if self.insertStrategy in FILE_STRATEGIES:
228
- tableOnly = [name for name in ('targetTableStage', 'preTargetAdhocQueries', 'postTargetAdhocQueries') if getattr(self, name)]
271
+ tableOnly = [name for name in ('targetTableStage', 'preTargetAdhocQueries', 'postTargetAdhocQueries', 'partitions')
272
+ if getattr(self, name) and not (name == 'partitions' and self.partitions is not None and self.partitions.column is None)]
229
273
  if tableOnly:
230
274
  raise ValueError('{} {} for a table in a database; insertStrategy: {} writes to files or Iceberg'.format(
231
275
  _listed(tableOnly), 'is' if len(tableOnly) == 1 else 'are', self.insertStrategy.value))
@@ -389,6 +433,9 @@ _TARGET_SETTINGS = {
389
433
  'targetTableStage': ('database',),
390
434
  'preTargetAdhocQueries': ('database',),
391
435
  'postTargetAdhocQueries': ('database',),
436
+ # A files or Iceberg target publishes what one writer wrote, and has no
437
+ # table for several to load into at once.
438
+ 'partitions': ('database',),
392
439
  }
393
440
 
394
441
  _TARGET_NAMES = {'database': 'a database', 'files': 'a files connection', 'iceberg': 'an Iceberg connection'}
@@ -426,6 +473,9 @@ def targetProblems(job: DataJobConfig, connection: Any) -> List[str]:
426
473
  what, _listed([strategy.value for strategy in _STRATEGIES[kind]]).replace(' and ', ' or '), job.insertStrategy.value))
427
474
 
428
475
  for setting, kinds in _TARGET_SETTINGS.items():
476
+ if setting == 'partitions' and job.partitions is not None and job.partitions.column is None:
477
+ # `partitions: auto` reads a lake target's job as one stream.
478
+ continue
429
479
  if getattr(job, setting) and kind not in kinds:
430
480
  problems.append('{} is for {}, and it is {}'.format(
431
481
  setting, ' or '.join(_TARGET_NAMES[other] for other in kinds), _TARGET_NAMES[kind]))
@@ -451,6 +501,31 @@ def targetProblems(job: DataJobConfig, connection: Any) -> List[str]:
451
501
  return problems
452
502
 
453
503
 
504
+ def partitionLimitProblems(job: DataJobConfig, connections: Mapping[str, ConnectionConfig]) -> List[str]:
505
+ """Why `job`'s partitions can't all run against its connections: each
506
+ partition opens a connection of its own to the source and the target, so
507
+ a partitioned job takes one of a connection's maxConcurrentJobs places for
508
+ each, and a job needing more places than there are could never start.
509
+ """
510
+
511
+ if job.partitions is None or job.partitions.automatic:
512
+ # An automatic count is fitted to what the connections have free.
513
+ return []
514
+
515
+ problems = []
516
+ for setting in ('sourceConnection', 'targetConnection'):
517
+ alias = getattr(job, setting)
518
+ connection = connections.get(alias)
519
+ limit = None if connection is None else connection.jobLimit()
520
+ if limit is not None and int(job.partitions.count) > limit:
521
+ why = ('DuckDB lets one process at a time open a file' if isinstance(connection, DuckDBConnection)
522
+ else 'its maxConcurrentJobs is {}'.format(limit))
523
+ problems.append('partitions.count is {}, and {} "{}" takes at most {} at once ({}); each partition holds one of its places, '
524
+ 'as a job does'.format(job.partitions.count, setting, alias, limit, why))
525
+
526
+ return problems
527
+
528
+
454
529
  class NameRuleConfig(BaseModel):
455
530
  """A discovery.yaml rule on column names."""
456
531
 
@@ -554,7 +629,7 @@ StorageLocation = Union[Annotated[str, Field(min_length=1)], TableLocation]
554
629
  # settings. A job that names any of these itself keeps its own value.
555
630
  DEFAULTABLE_JOB_FIELDS = frozenset({
556
631
  'active', 'refresh', 'sourceConnection', 'targetConnection', 'insertStrategy',
557
- 'chunkSize', 'retries', 'retryDelaySeconds', 'timeoutSeconds',
632
+ 'chunkSize', 'retries', 'retryDelaySeconds', 'timeoutSeconds', 'partitions',
558
633
  })
559
634
 
560
635
  # Within `masking`, the key alone. A key reference is not a policy: `columns`
@@ -633,6 +708,10 @@ class DataJobsFile(BaseModel):
633
708
  # as each job starts with the jobs running alongside it. See
634
709
  # masking.maskingThreadsFor and runner._runCycle.
635
710
  maskingThreads: Union[Literal['auto'], Annotated[int, Field(ge=1)]] = 1
711
+ # Whether a run stops before it starts, rather than masking in Python, when
712
+ # the native masker isn't installed or isn't this version. See
713
+ # masking.requireNativeProblem.
714
+ requireNative: bool = False
636
715
  # Tables no job copies, on purpose: connection alias -> table -> why. What
637
716
  # `bauta coverage` reads, so a table left out is a decision on the page
638
717
  # rather than something nobody noticed.
@@ -720,7 +799,9 @@ class Configuration:
720
799
  try:
721
800
  return schema.model_validate(rawConfiguration)
722
801
  except ValidationError as error:
723
- raise Configuration._invalid(error, sourceDescription) from error
802
+ # Not chained: pydantic's own error quotes the input it was given,
803
+ # a masking key or a password among it, in any traceback logged.
804
+ raise Configuration._invalid(error, sourceDescription) from None
724
805
 
725
806
 
726
807
  @staticmethod
@@ -752,7 +833,7 @@ class Configuration:
752
833
  try:
753
834
  return _CONNECTION_ADAPTER.validate_python(rawConnection)
754
835
  except ValidationError as error:
755
- raise Configuration._invalid(error, sourceDescription, skipLocation=1) from error
836
+ raise Configuration._invalid(error, sourceDescription, skipLocation=1) from None
756
837
 
757
838
 
758
839
  @staticmethod
@@ -841,6 +922,7 @@ class Configuration:
841
922
  jobName, job.sourceConnection, _TARGET_NAMES[targetKind(source)]))
842
923
  for problem in ([] if target is None else targetProblems(job, target)):
843
924
  problems.append('{}: targetConnection "{}" {}'.format(jobName, job.targetConnection, problem))
925
+ problems.extend('{}: {}'.format(jobName, problem) for problem in partitionLimitProblems(job, connections))
844
926
 
845
927
  if connectionAliases is not None and isinstance(job, DataJobConfig):
846
928
  if job.sourceConnection not in connectionAliases:
@@ -600,6 +600,57 @@ class Database:
600
600
  self.alter(query=query)
601
601
 
602
602
 
603
+ def copyKeys(self, fromTable: str, toTable: str) -> List[str]:
604
+ """Gives `toTable` the primary key and unique keys of `fromTable` that it
605
+ lacks, and says what it gave, as `primary key (id)` or `unique (email)`.
606
+ For a swap's stage table, which becomes the target, so the target keeps
607
+ its keys whichever table holds its name. See "How a swap works" in
608
+ docs/concepts/how-it-works.md.
609
+
610
+ Nothing is taken away, and a key `toTable` already has over the same
611
+ columns, in any order, is left as it is. Its columns are named as
612
+ `toTable` spells them, which must hold every one. Foreign keys are not
613
+ copied, for the reason `bauta schema` gives its stage tables none.
614
+ """
615
+
616
+ def folded(columns: Sequence[str]) -> frozenset:
617
+ return frozenset(column.upper() for column in columns)
618
+
619
+ primaryKey = self.dialect.primaryKey(self.cursor, fromTable)
620
+ uniqueKeys = self.dialect.uniqueKeys(self.cursor, fromTable)
621
+ ownPrimaryKey = self.dialect.primaryKey(self.cursor, toTable)
622
+ held = {folded(columns) for columns in self.dialect.uniqueKeys(self.cursor, toTable)}
623
+ if ownPrimaryKey:
624
+ held.add(folded(ownPrimaryKey))
625
+
626
+ addPrimaryKey = primaryKey if primaryKey and not ownPrimaryKey else []
627
+ if addPrimaryKey:
628
+ held.add(folded(addPrimaryKey))
629
+ addUnique = []
630
+ for columns in uniqueKeys:
631
+ if folded(columns) not in held:
632
+ held.add(folded(columns))
633
+ addUnique.append(columns)
634
+
635
+ if not addPrimaryKey and not addUnique:
636
+ return []
637
+
638
+ spelled = [self.catalogColumns(table=toTable, columns=list(columns)) for columns in [addPrimaryKey] + addUnique]
639
+ try:
640
+ self.dialect.addKeys(self.cursor, self.statementName(toTable), toTable, self.quoted(spelled[0]),
641
+ [self.quoted(columns) for columns in spelled[1:]])
642
+ self.connection.commit()
643
+ except BaseException:
644
+ self.connection.rollback()
645
+ raise
646
+ finally:
647
+ self.primaryKeyCache.pop(toTable, None)
648
+ self.columnNameCache.pop(toTable, None)
649
+
650
+ return ((['primary key ({})'.format(', '.join(spelled[0]))] if addPrimaryKey else [])
651
+ + ['unique ({})'.format(', '.join(columns)) for columns in spelled[1:]])
652
+
653
+
603
654
  def swap(self, targetTable: str, stageTable: str) -> None:
604
655
  """Exchanges the two tables by renaming, atomically everywhere but
605
656
  Oracle, through a temporary name in the stage table's schema.
@@ -5,6 +5,7 @@ from __future__ import annotations
5
5
 
6
6
  import contextlib
7
7
  import itertools
8
+ import re
8
9
  from abc import ABC, abstractmethod
9
10
  from enum import Enum
10
11
  from typing import Any, Dict, List, NamedTuple, Optional, Sequence, Tuple, Type, TypeVar
@@ -69,6 +70,54 @@ def _groupForeignKeys(rows: Sequence[Sequence[Any]]) -> List[ForeignKey]:
69
70
  ]
70
71
 
71
72
 
73
+ def _uniqueColumnGroups(rows: Sequence[Sequence[Any]]) -> List[Tuple[str, ...]]:
74
+ """Folds (keyName, column) rows, ordered by position within each key, into
75
+ each key's columns, leaving out a key with a NULL column -- an
76
+ expression, or a prefix of a column -- and a column list already seen.
77
+ """
78
+
79
+ grouped: Dict[str, List[Optional[str]]] = {}
80
+ for name, column in rows:
81
+ grouped.setdefault(name, []).append(column)
82
+
83
+ groups: List[Tuple[str, ...]] = []
84
+ for columns in grouped.values():
85
+ if any(column is None for column in columns):
86
+ continue
87
+ group = tuple(str(column) for column in columns)
88
+ if group not in groups:
89
+ groups.append(group)
90
+
91
+ return groups
92
+
93
+
94
+ _TABLE_NAME_PART = r'(?:"(?:[^"]|"")*"|\[[^\]]*\]|`(?:[^`]|``)*`|[^\s.(]+)'
95
+
96
+ # The head of a CREATE TABLE statement as SQLite and DuckDB keep it, up to the
97
+ # end of the table's name, schema-qualified or not.
98
+ _CREATE_TABLE_HEAD = re.compile(r'^\s*CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?{0}(?:\s*\.\s*{0})?'.format(_TABLE_NAME_PART), re.IGNORECASE)
99
+
100
+
101
+ def _withKeys(createStatement: str, table: str, primaryKey: Sequence[str], uniqueKeys: Sequence[Sequence[str]]) -> str:
102
+ """A table's own CREATE TABLE statement, naming it as `table` spells it,
103
+ with a primary key and unique constraints added after its last column.
104
+
105
+ The column list's closing parenthesis is the statement's last: what may
106
+ follow it -- WITHOUT ROWID, STRICT -- are words.
107
+ """
108
+
109
+ head = _CREATE_TABLE_HEAD.match(createStatement)
110
+ if head is None:
111
+ raise ConfigurationError('could not read the statement that created {}'.format(table))
112
+
113
+ statement = 'CREATE TABLE {}{}'.format(table, createStatement[head.end():]).rstrip().rstrip(';').rstrip()
114
+ closing = statement.rindex(')')
115
+ clauses = (['PRIMARY KEY ({})'.format(', '.join(primaryKey))] if primaryKey else []) + \
116
+ ['UNIQUE ({})'.format(', '.join(columns)) for columns in uniqueKeys]
117
+
118
+ return '{}, {}{}'.format(statement[:closing], ', '.join(clauses), statement[closing:])
119
+
120
+
72
121
  class ColumnDefinition(NamedTuple):
73
122
  """One column as the database's catalog describes it, for generating DDL.
74
123
 
@@ -335,6 +384,39 @@ class DatabaseDialect(ABC):
335
384
 
336
385
  return [row[0] for row in self._catalog(cursor, self.primaryKeyQuery(), table)]
337
386
 
387
+ def uniqueKeysQuery(self) -> str:
388
+ """One table's unique constraints and unique indexes, its primary key
389
+ aside, as rows of (name, column) ordered by name and position. A
390
+ column that is an expression, or only a prefix of one, comes back as
391
+ NULL, and its key is left out: copying it would take the database's
392
+ own DDL, not a column list.
393
+ """
394
+
395
+ raise NotImplementedError('{} cannot describe unique keys'.format(type(self).__name__))
396
+
397
+ def uniqueKeys(self, cursor: Cursor, table: str) -> List[Tuple[str, ...]]:
398
+ """The column lists of the table's unique keys over plain columns, each
399
+ once, its primary key aside. A partial or filtered one is left out by
400
+ each dialect's query, since it isn't unique over every row.
401
+ """
402
+
403
+ return _uniqueColumnGroups(self._catalog(cursor, self.uniqueKeysQuery(), table))
404
+
405
+ def addKeys(self, cursor: Cursor, table: str, catalogTable: str, primaryKey: Sequence[str], uniqueKeys: Sequence[Sequence[str]]) -> None:
406
+ """Gives `table`, as a statement names it, a primary key and unique
407
+ constraints over the quoted columns given; the caller commits.
408
+
409
+ Unnamed, so each database names them as it would any such constraint,
410
+ and no name can collide with one the swap's other table already has
411
+ -- the target's own, which keep their names when the tables trade
412
+ theirs. PostgreSQL, MySQL, MariaDB and Oracle take this as it is.
413
+ """
414
+
415
+ if primaryKey:
416
+ cursor.execute('ALTER TABLE {} ADD PRIMARY KEY ({})'.format(table, ', '.join(primaryKey)))
417
+ for columns in uniqueKeys:
418
+ cursor.execute('ALTER TABLE {} ADD UNIQUE ({})'.format(table, ', '.join(columns)))
419
+
338
420
  def columnDefinitions(self, cursor: Cursor, table: str) -> List[ColumnDefinition]:
339
421
 
340
422
  return _columnDefinitions(self._catalog(cursor, self.columnsQuery(), table))
@@ -10,7 +10,7 @@ from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, cast
10
10
 
11
11
  from ..driver import Connection, Cursor
12
12
  from ...configuration import ConfigurationError, DatabaseConfig, DatabaseType, DuckDBConnection
13
- from .base import ColumnCategory, ForeignKey, settingsOf, _OnConflictDialect, _renameInThreeSteps, _schemaForeignKeys
13
+ from .base import ColumnCategory, ForeignKey, settingsOf, _OnConflictDialect, _renameInThreeSteps, _schemaForeignKeys, _uniqueColumnGroups, _withKeys
14
14
  from .names import catalogTableName
15
15
 
16
16
 
@@ -344,6 +344,46 @@ class DuckDBDialect(_OnConflictDialect):
344
344
  "AND lower(table_name) = lower({}) AND database_name = current_database()")
345
345
 
346
346
 
347
+ def uniqueKeys(self, cursor: Cursor, table: str) -> List[Tuple[str, ...]]:
348
+ """UNIQUE constraints, which DuckDB lists apart from its primary key,
349
+ each with its columns as an array. A CREATE UNIQUE INDEX is no key a
350
+ swap could keep: DuckDB won't rename a table with an index.
351
+ """
352
+
353
+ schema, name = catalogTableName(self.databaseType, table)
354
+ cursor.execute("SELECT constraint_column_names FROM duckdb_constraints() WHERE constraint_type = 'UNIQUE' "
355
+ "AND lower(schema_name) = lower(COALESCE(?, current_schema())) AND lower(table_name) = lower(?) "
356
+ "AND database_name = current_database() ORDER BY constraint_index", (schema, name))
357
+
358
+ return _uniqueColumnGroups([(str(index), column) for index, (columns,) in enumerate(cursor.fetchall()) for column in columns])
359
+
360
+
361
+ def addKeys(self, cursor: Cursor, table: str, catalogTable: str, primaryKey: Sequence[str], uniqueKeys: Sequence[Sequence[str]]) -> None:
362
+ """DuckDB adds no UNIQUE constraint to a table, so the table is created
363
+ again from its own statement with the keys added, in the session's one
364
+ transaction. Only an empty table: a stage table just emptied for its
365
+ load.
366
+ """
367
+
368
+ schema, name = catalogTableName(self.databaseType, catalogTable)
369
+ where = ("lower(schema_name) = lower(COALESCE(?, current_schema())) AND lower(table_name) = lower(?) "
370
+ "AND database_name = current_database()")
371
+
372
+ cursor.execute('SELECT 1 FROM {} LIMIT 1'.format(table))
373
+ if cursor.fetchone() is not None:
374
+ raise ConfigurationError('{} holds rows, and DuckDB can only add a key by creating the table again'.format(catalogTable))
375
+
376
+ cursor.execute('SELECT sql FROM duckdb_tables() WHERE {}'.format(where), (schema, name))
377
+ created = cursor.fetchone()[0]
378
+ cursor.execute('SELECT sql FROM duckdb_indexes() WHERE {} AND sql IS NOT NULL'.format(where), (schema, name))
379
+ indexes = [row[0] for row in cursor.fetchall()]
380
+
381
+ cursor.execute('DROP TABLE {}'.format(table))
382
+ cursor.execute(_withKeys(created, table, primaryKey, uniqueKeys))
383
+ for index in indexes:
384
+ cursor.execute(index)
385
+
386
+
347
387
  def tableExistsQuery(self) -> str:
348
388
 
349
389
  return ("SELECT count(*) FROM information_schema.tables "