bauta 0.2.1__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. {bauta-0.2.1 → bauta-0.2.2}/PKG-INFO +3 -3
  2. {bauta-0.2.1 → bauta-0.2.2}/README.md +1 -1
  3. {bauta-0.2.1 → bauta-0.2.2}/bauta/configuration/models.py +2 -2
  4. {bauta-0.2.1 → bauta-0.2.2}/bauta/database/connection.py +18 -4
  5. {bauta-0.2.1 → bauta-0.2.2}/bauta/database/values.py +131 -28
  6. {bauta-0.2.1 → bauta-0.2.2}/bauta/jobs/pipeline.py +5 -4
  7. {bauta-0.2.1 → bauta-0.2.2}/bauta/jobs/runner.py +3 -2
  8. {bauta-0.2.1 → bauta-0.2.2}/bauta/lake/columnar.py +13 -6
  9. {bauta-0.2.1 → bauta-0.2.2}/bauta/lake/columns.py +20 -7
  10. {bauta-0.2.1 → bauta-0.2.2}/bauta/lake/iceberg.py +3 -2
  11. {bauta-0.2.1 → bauta-0.2.2}/bauta/masking/core.py +79 -11
  12. {bauta-0.2.1 → bauta-0.2.2}/bauta.egg-info/PKG-INFO +3 -3
  13. {bauta-0.2.1 → bauta-0.2.2}/bauta.egg-info/requires.txt +1 -1
  14. {bauta-0.2.1 → bauta-0.2.2}/pyproject.toml +2 -2
  15. {bauta-0.2.1 → bauta-0.2.2}/LICENSE +0 -0
  16. {bauta-0.2.1 → bauta-0.2.2}/bauta/__init__.py +0 -0
  17. {bauta-0.2.1 → bauta-0.2.2}/bauta/cli/__init__.py +0 -0
  18. {bauta-0.2.1 → bauta-0.2.2}/bauta/cli/common.py +0 -0
  19. {bauta-0.2.1 → bauta-0.2.2}/bauta/cli/generate.py +0 -0
  20. {bauta-0.2.1 → bauta-0.2.2}/bauta/cli/review.py +0 -0
  21. {bauta-0.2.1 → bauta-0.2.2}/bauta/cli/run.py +0 -0
  22. {bauta-0.2.1 → bauta-0.2.2}/bauta/configuration/__init__.py +0 -0
  23. {bauta-0.2.1 → bauta-0.2.2}/bauta/configuration/columnTypes.py +0 -0
  24. {bauta-0.2.1 → bauta-0.2.2}/bauta/configuration/connections.py +0 -0
  25. {bauta-0.2.1 → bauta-0.2.2}/bauta/configuration/environment.py +0 -0
  26. {bauta-0.2.1 → bauta-0.2.2}/bauta/database/__init__.py +0 -0
  27. {bauta-0.2.1 → bauta-0.2.2}/bauta/database/dialects/__init__.py +0 -0
  28. {bauta-0.2.1 → bauta-0.2.2}/bauta/database/dialects/base.py +0 -0
  29. {bauta-0.2.1 → bauta-0.2.2}/bauta/database/dialects/duckdb.py +0 -0
  30. {bauta-0.2.1 → bauta-0.2.2}/bauta/database/dialects/mssql.py +0 -0
  31. {bauta-0.2.1 → bauta-0.2.2}/bauta/database/dialects/mysql.py +0 -0
  32. {bauta-0.2.1 → bauta-0.2.2}/bauta/database/dialects/names.py +0 -0
  33. {bauta-0.2.1 → bauta-0.2.2}/bauta/database/dialects/oracle.py +0 -0
  34. {bauta-0.2.1 → bauta-0.2.2}/bauta/database/dialects/postgresql.py +0 -0
  35. {bauta-0.2.1 → bauta-0.2.2}/bauta/database/dialects/sqlite.py +0 -0
  36. {bauta-0.2.1 → bauta-0.2.2}/bauta/database/driver.py +0 -0
  37. {bauta-0.2.1 → bauta-0.2.2}/bauta/generate/__init__.py +0 -0
  38. {bauta-0.2.1 → bauta-0.2.2}/bauta/generate/builtinDiscovery.py +0 -0
  39. {bauta-0.2.1 → bauta-0.2.2}/bauta/generate/discovery.py +0 -0
  40. {bauta-0.2.1 → bauta-0.2.2}/bauta/generate/schema.py +0 -0
  41. {bauta-0.2.1 → bauta-0.2.2}/bauta/generate/subset.py +0 -0
  42. {bauta-0.2.1 → bauta-0.2.2}/bauta/generate/synthesize.py +0 -0
  43. {bauta-0.2.1 → bauta-0.2.2}/bauta/jobs/__init__.py +0 -0
  44. {bauta-0.2.1 → bauta-0.2.2}/bauta/jobs/dependencyGraph.py +0 -0
  45. {bauta-0.2.1 → bauta-0.2.2}/bauta/jobs/keys.py +0 -0
  46. {bauta-0.2.1 → bauta-0.2.2}/bauta/jobs/memory.py +0 -0
  47. {bauta-0.2.1 → bauta-0.2.2}/bauta/jobs/reporting.py +0 -0
  48. {bauta-0.2.1 → bauta-0.2.2}/bauta/jobs/targets.py +0 -0
  49. {bauta-0.2.1 → bauta-0.2.2}/bauta/jobs/workers.py +0 -0
  50. {bauta-0.2.1 → bauta-0.2.2}/bauta/lake/__init__.py +0 -0
  51. {bauta-0.2.1 → bauta-0.2.2}/bauta/lake/formats.py +0 -0
  52. {bauta-0.2.1 → bauta-0.2.2}/bauta/lake/stores.py +0 -0
  53. {bauta-0.2.1 → bauta-0.2.2}/bauta/lake/target.py +0 -0
  54. {bauta-0.2.1 → bauta-0.2.2}/bauta/log/__init__.py +0 -0
  55. {bauta-0.2.1 → bauta-0.2.2}/bauta/log/core.py +0 -0
  56. {bauta-0.2.1 → bauta-0.2.2}/bauta/log/scrubbing.py +0 -0
  57. {bauta-0.2.1 → bauta-0.2.2}/bauta/masking/__init__.py +0 -0
  58. {bauta-0.2.1 → bauta-0.2.2}/bauta/masking/fakeData.py +0 -0
  59. {bauta-0.2.1 → bauta-0.2.2}/bauta/masking/fpe.py +0 -0
  60. {bauta-0.2.1 → bauta-0.2.2}/bauta/masking/strategies.py +0 -0
  61. {bauta-0.2.1 → bauta-0.2.2}/bauta/py.typed +0 -0
  62. {bauta-0.2.1 → bauta-0.2.2}/bauta/review/__init__.py +0 -0
  63. {bauta-0.2.1 → bauta-0.2.2}/bauta/review/audit.py +0 -0
  64. {bauta-0.2.1 → bauta-0.2.2}/bauta/review/coverage.py +0 -0
  65. {bauta-0.2.1 → bauta-0.2.2}/bauta/review/references.py +0 -0
  66. {bauta-0.2.1 → bauta-0.2.2}/bauta/transform/__init__.py +0 -0
  67. {bauta-0.2.1 → bauta-0.2.2}/bauta/transform/builtinTransforms.py +0 -0
  68. {bauta-0.2.1 → bauta-0.2.2}/bauta/transform/core.py +0 -0
  69. {bauta-0.2.1 → bauta-0.2.2}/bauta.egg-info/SOURCES.txt +0 -0
  70. {bauta-0.2.1 → bauta-0.2.2}/bauta.egg-info/dependency_links.txt +0 -0
  71. {bauta-0.2.1 → bauta-0.2.2}/bauta.egg-info/entry_points.txt +0 -0
  72. {bauta-0.2.1 → bauta-0.2.2}/bauta.egg-info/top_level.txt +0 -0
  73. {bauta-0.2.1 → bauta-0.2.2}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: bauta
3
- Version: 0.2.1
3
+ Version: 0.2.2
4
4
  Summary: Safe, realistic copies of production across databases: masking, subsets, synthetic data, and the ETL to move them
5
5
  Author: David Ribeiro
6
6
  License-Expression: MIT
@@ -55,7 +55,7 @@ Requires-Dist: cryptography>=42; extra == "fpe"
55
55
  Provides-Extra: all
56
56
  Requires-Dist: bauta[duckdb,files,fpe,iceberg,mariadb,mssql,mysql,oracle,postgresql,sqlite]; extra == "all"
57
57
  Provides-Extra: native
58
- Requires-Dist: bauta-rs==0.2.1; extra == "native"
58
+ Requires-Dist: bauta-rs==0.2.2; extra == "native"
59
59
  Provides-Extra: dev
60
60
  Requires-Dist: pytest==9.1.1; extra == "dev"
61
61
  Requires-Dist: mypy==2.3.1; extra == "dev"
@@ -107,7 +107,7 @@ pip install "bauta[postgresql,oracle]"
107
107
  | `native` | `bauta-rs`, the native masker (below) | nothing on Linux (x86-64, ARM) or macOS; elsewhere, [Rust](https://rustup.rs) 1.83 or newer |
108
108
  | `all` | every driver above | nothing |
109
109
 
110
- **The native masker (optional).** `bauta-rs` masks in Rust: about ten times the throughput on one core, with identical masks. It can also mask on several cores: `jobs.yaml`'s `maskingThreads` is `1` by default, a number up to the cores available, or `auto` to divide the cores between the jobs running (see [masking threads](docs/masking.md#masking-threads)). `pip install "bauta[postgresql,native]"` installs the version that matches, which is the only one Bauta uses. Without it, everything works, only slower. See [the native masker](docs/masking.md#the-native-masker).
110
+ **The native masker (optional).** `bauta-rs` masks in Rust: about ten times the throughput on one core, with identical masks. It can also mask on several cores: `jobs.yaml`'s `maskingThreads` is `1` by default, a number up to the cores available, or `auto` to divide half the cores between the jobs running (see [masking threads](docs/masking.md#masking-threads)). `pip install "bauta[postgresql,native]"` installs the version that matches, which is the only one Bauta uses. Without it, everything works, only slower. See [the native masker](docs/masking.md#the-native-masker).
111
111
 
112
112
 
113
113
  ## Quickstart
@@ -39,7 +39,7 @@ pip install "bauta[postgresql,oracle]"
39
39
  | `native` | `bauta-rs`, the native masker (below) | nothing on Linux (x86-64, ARM) or macOS; elsewhere, [Rust](https://rustup.rs) 1.83 or newer |
40
40
  | `all` | every driver above | nothing |
41
41
 
42
- **The native masker (optional).** `bauta-rs` masks in Rust: about ten times the throughput on one core, with identical masks. It can also mask on several cores: `jobs.yaml`'s `maskingThreads` is `1` by default, a number up to the cores available, or `auto` to divide the cores between the jobs running (see [masking threads](docs/masking.md#masking-threads)). `pip install "bauta[postgresql,native]"` installs the version that matches, which is the only one Bauta uses. Without it, everything works, only slower. See [the native masker](docs/masking.md#the-native-masker).
42
+ **The native masker (optional).** `bauta-rs` masks in Rust: about ten times the throughput on one core, with identical masks. It can also mask on several cores: `jobs.yaml`'s `maskingThreads` is `1` by default, a number up to the cores available, or `auto` to divide half the cores between the jobs running (see [masking threads](docs/masking.md#masking-threads)). `pip install "bauta[postgresql,native]"` installs the version that matches, which is the only one Bauta uses. Without it, everything works, only slower. See [the native masker](docs/masking.md#the-native-masker).
43
43
 
44
44
 
45
45
  ## Quickstart
@@ -619,8 +619,8 @@ class DataJobsFile(BaseModel):
619
619
  history: Optional[StorageLocation] = None
620
620
  manifest: Optional[StorageLocation] = None
621
621
  # Threads the native masker spreads each job's chunks over: one by default,
622
- # a number up to the cores available, or `auto`, which shares the cores as
623
- # each job starts with the jobs running alongside it. See
622
+ # a number up to the cores available, or `auto`, which shares half the cores
623
+ # as each job starts with the jobs running alongside it. See
624
624
  # masking.maskingThreadsFor and runner._runCycle.
625
625
  maskingThreads: Union[Literal['auto'], Annotated[int, Field(ge=1)]] = 1
626
626
  # Tables no job copies, on purpose: connection alias -> table -> why. What
@@ -4,7 +4,7 @@ from types import TracebackType
4
4
  from typing import Any, Dict, Iterator, List, Optional, Sequence, Set, Tuple, Type
5
5
 
6
6
  from .driver import Connection, Cursor
7
- from .values import WANTS_COLUMN_TYPES, prepareParameters, prepareValues
7
+ from .values import WANTS_COLUMN_TYPES, ValuePreparer, prepareParameters
8
8
  from ..configuration import WATERMARK_PLACEHOLDER, ConfigurationError, ConnectionConfig, DatabaseConfig, DatabaseType, FilesConnection, IcebergConnection
9
9
  from .dialects import ColumnDefinition, DatabaseDialect, DuckDBDialect, ForeignKey, MariaDBDialect, MSSQLDialect, MySQLDialect, OracleDialect, PostgreSQLDialect, \
10
10
  SQLiteDialect, catalogName, quoteFoldedTable, quoteIdentifier, splitTableName, suffixedName, tooLongName
@@ -113,6 +113,8 @@ class Database:
113
113
  self.primaryKeyCache: Dict[str, List[str]] = {}
114
114
  self.columnNameCache: Dict[str, List[str]] = {}
115
115
  self.columnTypeCache: Dict[str, Dict[str, Any]] = {}
116
+ # One per table loaded, so each learns from its table's chunks.
117
+ self._preparers: Dict[str, ValuePreparer] = {}
116
118
  self._streams: Set[RowStream] = set()
117
119
  self.connect()
118
120
 
@@ -480,6 +482,14 @@ class Database:
480
482
  yield data[index:index + chunkSize]
481
483
 
482
484
 
485
+ def _preparer(self, table: str) -> ValuePreparer:
486
+
487
+ if table not in self._preparers:
488
+ self._preparers[table] = ValuePreparer(self.type)
489
+
490
+ return self._preparers[table]
491
+
492
+
483
493
  def insert(self, table: str, data: List[Tuple[Any, ...]], chunkSize: int = 100, columns: Optional[List[str]] = None) -> None:
484
494
  """`columns` defaults to all of the table's, in its order; `data` must
485
495
  match. Each batch uses the dialect's bulk path where it has one, and
@@ -493,8 +503,9 @@ class Database:
493
503
  query = 'INSERT INTO {} ({}) VALUES ({})'.format(
494
504
  statementTable, ', '.join(resolvedColumns), ', '.join(self.dialect.placeholders(len(resolvedColumns))))
495
505
 
506
+ preparer = self._preparer(table)
496
507
  for batch in self._batches(data, chunkSize):
497
- batch = prepareValues(self.type, batch, columnTypes)
508
+ batch = preparer.prepare(batch, columnTypes)
498
509
  if not self.dialect.bulkInsert(self.cursor, statementTable, resolvedColumns, batch):
499
510
  self.cursor.executemany(query, batch)
500
511
  self.connection.commit()
@@ -510,15 +521,18 @@ class Database:
510
521
 
511
522
  normalizedColumns = [column.upper() for column in allColumns]
512
523
  keyIndexes = [normalizedColumns.index(column.upper()) for column in primaryKeyColumns if column.upper() in normalizedColumns]
513
- canCollapse = len(keyIndexes) == len(primaryKeyColumns)
524
+ # Only a dialect with a bulk path loads the collapsed batch; for the
525
+ # rest, collapsing it was work thrown away.
526
+ canCollapse = len(keyIndexes) == len(primaryKeyColumns) and type(self.dialect).bulkUpsert is not DatabaseDialect.bulkUpsert
514
527
 
515
528
  allColumns, primaryKeyColumns, nonPrimaryKeyColumns = self.quoted(allColumns), self.quoted(primaryKeyColumns), self.quoted(nonPrimaryKeyColumns)
516
529
  statementTable = self.statementName(table)
517
530
  query = self.dialect.upsertQuery(table=statementTable, allColumns=allColumns, primaryKeyColumns=primaryKeyColumns,
518
531
  nonPrimaryKeyColumns=nonPrimaryKeyColumns)
519
532
 
533
+ preparer = self._preparer(table)
520
534
  for batch in self._batches(data, chunkSize):
521
- batch = prepareValues(self.type, batch, columnTypes)
535
+ batch = preparer.prepare(batch, columnTypes)
522
536
  loaded = False
523
537
  if canCollapse:
524
538
  lastPerKey = list({tuple(row[index] for index in keyIndexes): row for row in batch}.values())
@@ -12,7 +12,8 @@ import decimal
12
12
  import itertools
13
13
  import json
14
14
  import uuid
15
- from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple
15
+ from operator import itemgetter
16
+ from typing import Any, Callable, Dict, List, Optional, Sequence, Set, Tuple
16
17
 
17
18
  from ..configuration import DatabaseType
18
19
 
@@ -155,36 +156,138 @@ def conversionFor(databaseType: DatabaseType, kind: type) -> Optional[Conversion
155
156
  return _found[key]
156
157
 
157
158
 
158
- def prepareValues(databaseType: DatabaseType, rows: List[Tuple[Any, ...]], columnTypes: Optional[Sequence[Any]] = None) -> List[Tuple[Any, ...]]:
159
- """`rows` as `databaseType`'s driver must receive them.
159
+ _NONE = type(None)
160
+
161
+ # Rows to each distinct type signature below which finding a chunk's columns
162
+ # by their signatures beats transposing it: NULLs scattered over many columns
163
+ # make most rows' signatures distinct.
164
+ _ROWS_PER_SIGNATURE = 8
165
+
166
+
167
+ def _columnKinds(rows: List[Tuple[Any, ...]]) -> Optional[Tuple[List[List[type]], List[bool]]]:
168
+ """Each column's exact types, and whether its ints all fit in 64 bits,
169
+ from the native extension in one pass; None without it, or for rows it
170
+ doesn't read, which the caller then reads in Python.
171
+ """
172
+
173
+ from ..masking.core import nativeExtension
174
+
175
+ native = nativeExtension()
176
+
177
+ return native.columnKinds(rows) if native is not None else None
178
+
179
+
180
+ def _fitIn64Bits(column: Sequence[Any], kinds: Set[type]) -> bool:
181
+ """Whether a column of ints, and perhaps NULLs, needs no _wideInteger: its
182
+ smallest and largest, found in C, stand for the rest.
183
+ """
184
+
185
+ values = column if _NONE not in kinds else [value for value in column if value is not None]
186
+
187
+ return not values or (min(values) in _BOUND_INTEGERS and max(values) in _BOUND_INTEGERS)
188
+
189
+
190
+ class ValuePreparer:
191
+ """prepareValues for the chunks of one load. What it learns of one chunk
192
+ changes how much of the next it looks at, never what it returns.
193
+
194
+ Which columns hold a value to convert comes from each column's exact
195
+ types, which the native extension finds in one pass where it is in use.
196
+ In Python, a chunk usually either holds nothing to convert, which one pass
197
+ over its values in C finds, or holds the same kinds in the same columns as
198
+ the chunk before -- an integer id bound for SQLite, say. For those it skips
199
+ that pass, and finds the columns from the rows' distinct type signatures,
200
+ another pass in C, rather than by transposing the chunk and scanning every
201
+ column. Where NULLs make the signatures too many, it transposes, then and
202
+ for the rest of the load.
203
+
204
+ A column of ints within 64 bits, and NULLs, needs nothing on the two
205
+ databases that convert ints, which is nearly every such column: it is
206
+ checked whole, not value by value.
207
+ """
208
+
209
+ def __init__(self, databaseType: DatabaseType) -> None:
210
+ self.databaseType = databaseType
211
+ # Whether the last chunk held a value a conversion applies to.
212
+ self._converting = False
213
+ self._bySignature = True
160
214
 
161
- Every chunk of every load comes through here, so it does as little as it
162
- can: the distinct types in the chunk are gathered in C, and a chunk with
163
- none to convert is returned as it is. Otherwise only the columns holding
164
- one are converted, and the rest carried through untouched.
215
+
216
+ def _kindsInPython(self, rows: List[Tuple[Any, ...]]) -> Tuple[Optional[List[Set[type]]], Optional[List[Tuple[Any, ...]]]]:
217
+ """Each column's exact types, and the chunk transposed where that was
218
+ the way to find them; no types where the chunk holds nothing to
219
+ convert.
220
+ """
221
+
222
+ if not self._converting and not any(conversionFor(self.databaseType, kind) for kind in set(map(type, itertools.chain.from_iterable(rows)))):
223
+ return None, None
224
+
225
+ if self._bySignature:
226
+ signatures = set(map(tuple, map(map, itertools.repeat(type), rows)))
227
+ if len(signatures) * _ROWS_PER_SIGNATURE <= len(rows):
228
+ return [set(kinds) for kinds in zip(*signatures)], None
229
+ self._bySignature = False
230
+
231
+ columns = list(zip(*rows))
232
+
233
+ return [set(map(type, column)) for column in columns], columns
234
+
235
+
236
+ def prepare(self, rows: List[Tuple[Any, ...]], columnTypes: Optional[Sequence[Any]] = None) -> List[Tuple[Any, ...]]:
237
+
238
+ if not rows:
239
+ return rows
240
+
241
+ columns: Optional[List[Tuple[Any, ...]]] = None
242
+ fits: Optional[List[bool]] = None
243
+ kindsByColumn: Optional[List[Set[type]]]
244
+ found = _columnKinds(rows)
245
+ if found is not None:
246
+ kindsByColumn, fits = [set(kinds) for kinds in found[0]], found[1]
247
+ else:
248
+ kindsByColumn, columns = self._kindsInPython(rows)
249
+ if kindsByColumn is None:
250
+ return rows
251
+
252
+ self._converting = False
253
+ converted: Dict[int, Tuple[Any, ...]] = {}
254
+ for index, kinds in enumerate(kindsByColumn):
255
+ conversions = {kind: conversionFor(self.databaseType, kind) for kind in kinds}
256
+ applying = {conversion for conversion in conversions.values() if conversion is not None}
257
+ if not applying:
258
+ continue
259
+ self._converting = True
260
+
261
+ column = columns[index] if columns is not None else tuple(map(itemgetter(index), rows))
262
+ if applying == {_wideInteger} and kinds <= {int, _NONE} and (fits[index] if fits is not None else _fitIn64Bits(column, kinds)):
263
+ continue
264
+
265
+ columnType = columnTypes[index] if columnTypes is not None and index < len(columnTypes) else None
266
+ values = tuple(value if (conversion := conversions[type(value)]) is None else conversion(value, columnType) for value in column)
267
+ # A conversion may leave every value as it was, and the column is
268
+ # then kept as it is.
269
+ if any(new is not old for new, old in zip(values, column)):
270
+ converted[index] = values
271
+
272
+ if not converted:
273
+ return rows
274
+
275
+ if columns is None:
276
+ columns = list(zip(*rows))
277
+ for index, values in converted.items():
278
+ columns[index] = values
279
+
280
+ return list(zip(*columns))
281
+
282
+
283
+ def prepareValues(databaseType: DatabaseType, rows: List[Tuple[Any, ...]], columnTypes: Optional[Sequence[Any]] = None) -> List[Tuple[Any, ...]]:
284
+ """`rows` as `databaseType`'s driver must receive them: a chunk with
285
+ nothing to convert as it is, and otherwise only the columns holding a
286
+ value to convert converted, the rest carried through untouched. A load of
287
+ many chunks keeps a ValuePreparer instead.
165
288
  """
166
289
 
167
- if not rows:
168
- return rows
169
-
170
- if all(conversionFor(databaseType, kind) is None for kind in set(map(type, itertools.chain.from_iterable(rows)))):
171
- return rows
172
-
173
- columns = list(zip(*rows))
174
- changed = False
175
- for index, column in enumerate(columns):
176
- conversions = {kind: conversionFor(databaseType, kind) for kind in set(map(type, column))}
177
- if not any(conversions.values()):
178
- continue
179
- columnType = columnTypes[index] if columnTypes is not None and index < len(columnTypes) else None
180
- converted = tuple(value if (conversion := conversions[type(value)]) is None else conversion(value, columnType) for value in column)
181
- # A conversion may leave every value as it was -- integers, nearly
182
- # always within 64 bits -- and the chunk is then kept whole.
183
- if any(new is not old for new, old in zip(converted, column)):
184
- columns[index] = converted
185
- changed = True
186
-
187
- return list(zip(*columns)) if changed else rows
290
+ return ValuePreparer(databaseType).prepare(rows, columnTypes)
188
291
 
189
292
 
190
293
  def prepareParameters(databaseType: DatabaseType, parameters: Sequence[Any]) -> Tuple[Any, ...]:
@@ -28,10 +28,11 @@ from .targets import LoadTarget, PostLoadError, TableTarget
28
28
  logger = logging.getLogger(LOGGER_NAME)
29
29
 
30
30
 
31
- # How many chunks may be masked ahead of the one being written. One already
32
- # keeps the reader, masker and writer all busy, holding three chunks at once;
33
- # more buys no overlap and costs a chunk of memory each.
34
- PIPELINE_DEPTH = 1
31
+ # How many chunks may be masked ahead of the one being written. One keeps the
32
+ # reader, masker and writer all busy on average, holding three chunks at once;
33
+ # a second absorbs a slow chunk on either side when masking and the database
34
+ # take about as long, for one more chunk of memory. More buys nothing further.
35
+ PIPELINE_DEPTH = 2
35
36
 
36
37
 
37
38
  # Caps the doubling backoff, which would otherwise wait 5.7 hours in all over
@@ -175,8 +175,9 @@ def _runCycle(dependencyGraph: DependencyGraph, workers: int, connectionConfigur
175
175
 
176
176
  `maskingThreads` is shared out as each job starts, between it and the jobs
177
177
  that will run alongside it: those already running and those starting with
178
- it. So the last job of a cycle, running alone, gets every core. A running
179
- job keeps its share; cores freed after it started go to the next to start.
178
+ it. So the last job of a cycle, running alone, gets the whole of `auto`'s
179
+ half of the cores. A running job keeps its share; cores freed after it
180
+ started go to the next to start.
180
181
  """
181
182
 
182
183
  running: List[_JobProcess] = []
@@ -7,12 +7,13 @@ from __future__ import annotations
7
7
  import datetime
8
8
  import logging
9
9
  import secrets
10
+ from operator import itemgetter
10
11
  from typing import Any, Dict, List, Optional, Sequence, Tuple
11
12
 
12
13
  from ..configuration import ColumnType, ConfigurationError, DataJobConfig, parseColumnType
13
14
  from ..jobs.targets import LoadTarget
14
15
  from ..log import LOGGER_NAME
15
- from .columns import arrowType, inferType, normalized, reportedDecimal, toArrow
16
+ from .columns import PLAIN_KINDS, arrowType, inferType, normalized, reportedDecimal, toArrow
16
17
 
17
18
  logger = logging.getLogger(LOGGER_NAME)
18
19
 
@@ -113,10 +114,16 @@ class ColumnarTarget(LoadTarget):
113
114
 
114
115
  arrays = []
115
116
  for index, column in enumerate(self.columns):
116
- raw = [row[index] for row in rows]
117
- if not column.documents and any(isinstance(value, (dict, list)) for value in raw):
118
- column.documents = True
119
- values = [normalized(value) for value in raw]
117
+ raw = list(map(itemgetter(index), rows))
118
+ kinds = set(map(type, raw))
119
+ exactKinds = kinds if kinds <= PLAIN_KINDS else None
120
+ if exactKinds is not None:
121
+ # Nothing a file holds differently, and no documents.
122
+ values = raw
123
+ else:
124
+ if not column.documents and any(isinstance(value, (dict, list)) for value in raw):
125
+ column.documents = True
126
+ values = [normalized(value) for value in raw]
120
127
  if column.type is None:
121
128
  inferred = inferType(column.name, values, column.reported)
122
129
  column.type = None if inferred is None else self._settled(inferred)
@@ -126,7 +133,7 @@ class ColumnarTarget(LoadTarget):
126
133
  if column.type is None:
127
134
  arrays.append(pyarrow.nulls(len(values)))
128
135
  else:
129
- arrays.append(toArrow(column.name, values, column.type, column.lenient))
136
+ arrays.append(toArrow(column.name, values, column.type, column.lenient, exactKinds))
130
137
 
131
138
  self._buffered.append(arrays)
132
139
  self._bufferedBytes += sum(array.nbytes for array in arrays)
@@ -108,6 +108,13 @@ def _kind(value: Any) -> type:
108
108
  return type(value)
109
109
 
110
110
 
111
+ # The exact types a value can have for normalized() to leave it as it is and
112
+ # _kind() to name it as itself. A column holding nothing else -- nearly every
113
+ # column -- needs neither called on each value, which is most of the time a
114
+ # chunk takes to become Arrow; the column's exact types, gathered in C, say so.
115
+ PLAIN_KINDS = frozenset(_KINDS) | {type(None)}
116
+
117
+
111
118
  def _digits(value: int) -> int:
112
119
 
113
120
  return len(str(abs(value)))
@@ -255,19 +262,22 @@ def _accepts(columnType: ColumnType, lenient: bool) -> _Accepts:
255
262
  raise AssertionError('no kind {}'.format(kind))
256
263
 
257
264
 
258
- def toArrow(column: str, values: List[Any], columnType: ColumnType, lenient: bool) -> Any:
265
+ def toArrow(column: str, values: List[Any], columnType: ColumnType, lenient: bool, exactKinds: Optional[Set[type]] = None) -> Any:
259
266
  """`values`, normalized, as an Arrow array of the column's type; a
260
- FileTypeError naming the column when one doesn't fit.
267
+ FileTypeError naming the column when one doesn't fit. `exactKinds` is
268
+ `set(map(type, values))` where that is within PLAIN_KINDS, which saves
269
+ telling each value's kind.
261
270
  """
262
271
 
263
272
  import pyarrow
264
273
 
265
274
  accepts = _accepts(columnType, lenient)
266
- kinds = {_kind(value) for value in values if value is not None}
267
- if columnType.kind == 'timestamp' and any(isinstance(value, datetime.datetime) and value.tzinfo is not None for value in values):
275
+ kindOf = _kind if exactKinds is None else type
276
+ kinds = {_kind(value) for value in values if value is not None} if exactKinds is None else exactKinds - {type(None)}
277
+ if columnType.kind == 'timestamp' and datetime.datetime in kinds and any(isinstance(value, datetime.datetime) and value.tzinfo is not None for value in values):
268
278
  raise FileTypeError('column {} is written as timestamp, without a time zone, and a row holds one with; declare it '
269
279
  'timestamptz in targetColumnTypes'.format(column))
270
- if columnType.kind == 'timestamptz' and any(isinstance(value, datetime.datetime) and value.tzinfo is None for value in values):
280
+ if columnType.kind == 'timestamptz' and datetime.datetime in kinds and any(isinstance(value, datetime.datetime) and value.tzinfo is None for value in values):
271
281
  raise FileTypeError('column {} is written as timestamptz, and a row holds a time without a time zone, which could be '
272
282
  'any instant; declare it timestamp in targetColumnTypes'.format(column))
273
283
  if columnType.kind == 'date' and datetime.datetime in kinds:
@@ -281,12 +291,15 @@ def toArrow(column: str, values: List[Any], columnType: ColumnType, lenient: boo
281
291
 
282
292
  conversions = {kind: accepts[kind] for kind in kinds if accepts[kind] is not None}
283
293
  if conversions:
284
- values = [value if value is None or _kind(value) not in conversions else conversions[_kind(value)](value) # type: ignore[misc]
294
+ values = [value if value is None or kindOf(value) not in conversions else conversions[kindOf(value)](value) # type: ignore[misc]
285
295
  for value in values]
286
296
 
287
297
  if isInteger(columnType):
288
298
  low, high = _INTEGER_RANGES[columnType.kind]
289
- if any(value is not None and not low <= value < high for value in values):
299
+ # Bools are ints by now; the smallest and largest, found in C, stand
300
+ # for the rest.
301
+ present = values if exactKinds is not None and type(None) not in exactKinds else [value for value in values if value is not None]
302
+ if present and (min(present) < low or max(present) >= high):
290
303
  raise FileTypeError('column {} is written as {}, and a row holds an integer outside its range; declare a wider '
291
304
  'type in targetColumnTypes, such as int64 or decimal(38,0)'.format(column, columnType))
292
305
 
@@ -32,8 +32,9 @@ try:
32
32
  from pyiceberg.io.pyarrow import PyArrowFileIO
33
33
  except ImportError:
34
34
  # The extra isn't installed: requirePyiceberg() says so, with how to
35
- # install it, when a job first needs a catalog.
36
- PyArrowFileIO = object # type: ignore[misc,assignment]
35
+ # install it, when a job first needs a catalog. The ignore is needed only
36
+ # where pyiceberg is, whose class this replaces.
37
+ PyArrowFileIO = object # type: ignore[misc,assignment,unused-ignore]
37
38
 
38
39
  logger = logging.getLogger(LOGGER_NAME)
39
40
 
@@ -88,6 +88,14 @@ def _nativeModule() -> Any:
88
88
  return bauta_rs
89
89
 
90
90
 
91
+ def nativeExtension() -> Any:
92
+ """The native extension where it is in use, as nativeVersion() says, or
93
+ None: for the rest of the package, which uses it beyond masking.
94
+ """
95
+
96
+ return _nativeModule()
97
+
98
+
91
99
  def nativeVersion() -> Optional[str]:
92
100
  """The native masker's version, or None when masking runs in pure Python."""
93
101
 
@@ -143,15 +151,19 @@ def effectiveMaskingThreads(setting: Union[str, int]) -> Union[str, int]:
143
151
 
144
152
 
145
153
  def maskingThreadsFor(setting: Union[str, int], concurrentJobs: int) -> int:
146
- """How many threads a job masks with: a number as it is, or for `auto` the
147
- cores shared out between `concurrentJobs` -- a starting job and the ones
148
- running alongside it. BAUTA_MASKING_THREADS overrides the setting, which is
149
- checked by effectiveMaskingThreads.
154
+ """How many threads a job masks with: a number as it is, or for `auto` half
155
+ the cores, shared out between `concurrentJobs` -- a starting job and the
156
+ ones running alongside it. BAUTA_MASKING_THREADS overrides the setting,
157
+ which is checked by effectiveMaskingThreads.
158
+
159
+ Half, because masking only has to keep up with the job's reading and
160
+ writing, which need cores of their own: a wide table on ten cores copied
161
+ as fast with four masking threads as with six or eight, and slower with ten.
150
162
  """
151
163
 
152
164
  setting = effectiveMaskingThreads(setting)
153
165
  if setting == 'auto':
154
- return max(1, availableCores() // max(1, concurrentJobs))
166
+ return max(1, availableCores() // 2 // max(1, concurrentJobs))
155
167
 
156
168
  return int(setting)
157
169
 
@@ -445,16 +457,22 @@ class Strategy:
445
457
  masked, problems = self._native.maskColumn(values)
446
458
 
447
459
  for index in sorted(problems):
448
- problem = problems[index]
449
- if problem == _NATIVE_FALLBACK:
450
- value = values[index]
451
- masked[index] = None if value is None else self.mask(value)
452
- else:
453
- raise MaskingError(problem[1])
460
+ masked[index] = self._finishNatively(values[index], problems[index])
454
461
 
455
462
  return masked
456
463
 
457
464
 
465
+ def _finishNatively(self, value: Any, problem: Any) -> Any:
466
+ """A value the extension handed back: masked here, or refused with the
467
+ extension's message.
468
+ """
469
+
470
+ if problem == _NATIVE_FALLBACK:
471
+ return None if value is None else self.mask(value)
472
+
473
+ raise MaskingError(problem[1])
474
+
475
+
458
476
  def _maskRemembered(self, value: Any) -> Any:
459
477
  """mask(), from the cache where the value's type allows."""
460
478
 
@@ -659,6 +677,12 @@ class BoundMasking:
659
677
  # and past that once a third or more of them are masked.
660
678
  width = len(self.strategies)
661
679
  self._transposeToSplice = width <= 20 or 3 * len(self._maskedIndexes) >= width
680
+ # The extension's masker for each column it masks with the rest of the
681
+ # chunk, in one call; None for a column kept, or masked in Python. A
682
+ # strategy that masks its column its own way keeps doing so.
683
+ self._natives = [strategy._native if not strategy.PASSTHROUGH and type(strategy).maskColumn is Strategy.maskColumn else None
684
+ for strategy in self.strategies]
685
+ self._chunkNatively = any(native is not None for native in self._natives)
662
686
 
663
687
 
664
688
  def _maskColumn(self, index: int, values: Sequence[Any], chunkIndex: int) -> List[Any]:
@@ -689,6 +713,9 @@ class BoundMasking:
689
713
  raise MaskingError('the policy covers {} column(s) and these rows have {}; a bound plan is applied to the rows of the '
690
714
  'query it was bound to'.format(len(self.strategies), len(rows[0])))
691
715
 
716
+ if self._chunkNatively:
717
+ return self._applyNatively(rows, chunkIndex)
718
+
692
719
  if self._transposeToSplice:
693
720
  # Rows rebuilt from columns in C, each column masked as the
694
721
  # transpose yields it.
@@ -709,6 +736,47 @@ class BoundMasking:
709
736
  return spliced
710
737
 
711
738
 
739
+ def _applyNatively(self, rows: Sequence[Tuple[Any, ...]], chunkIndex: int) -> List[Tuple[Any, ...]]:
740
+ """The chunk through the extension in one call, every column it covers
741
+ masked at once. Python then goes through the columns in read order,
742
+ finishing the values the extension handed back and masking the columns
743
+ it doesn't cover, so the first bad value raises as it would column by
744
+ column.
745
+ """
746
+
747
+ masked, problems = _nativeModule().maskChunk(self._natives, rows)
748
+
749
+ # In row order, so each column's come out in row order too.
750
+ handedBack: Dict[int, List[Tuple[int, Any]]] = {}
751
+ for index, position, problem in problems:
752
+ handedBack.setdefault(index, []).append((position, problem))
753
+
754
+ columns: Dict[int, List[Any]] = {}
755
+ finished: Dict[int, List[Tuple[int, Any]]] = {}
756
+ for index in self._maskedIndexes:
757
+ if self._natives[index] is None:
758
+ columns[index] = self._maskColumn(index, [row[index] for row in rows], chunkIndex)
759
+ continue
760
+ for position, problem in handedBack.get(index, ()):
761
+ try:
762
+ value = self.strategies[index]._finishNatively(rows[position][index], problem)
763
+ except MaskingError as error:
764
+ raise MaskingError('column "{}": {}'.format(self.manifest[index].column, error)) from None
765
+ finished.setdefault(position, []).append((index, value))
766
+
767
+ if columns or finished:
768
+ # Every row when Python masked a column, else only those it finished.
769
+ for position in (range(len(masked)) if columns else sorted(finished)):
770
+ values = list(masked[position])
771
+ for index, column in columns.items():
772
+ values[index] = column[position]
773
+ for index, value in finished.get(position, ()):
774
+ values[index] = value
775
+ masked[position] = tuple(values)
776
+
777
+ return masked
778
+
779
+
712
780
  def buildMaskingManifest(outcomes: Sequence[Any], declared: Mapping[str, Mapping[str, Any]],
713
781
  generatedAt: Optional[datetime.datetime] = None) -> Dict[str, Any]:
714
782
  """What was masked, how, and under which key, for an auditor. Never a value
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: bauta
3
- Version: 0.2.1
3
+ Version: 0.2.2
4
4
  Summary: Safe, realistic copies of production across databases: masking, subsets, synthetic data, and the ETL to move them
5
5
  Author: David Ribeiro
6
6
  License-Expression: MIT
@@ -55,7 +55,7 @@ Requires-Dist: cryptography>=42; extra == "fpe"
55
55
  Provides-Extra: all
56
56
  Requires-Dist: bauta[duckdb,files,fpe,iceberg,mariadb,mssql,mysql,oracle,postgresql,sqlite]; extra == "all"
57
57
  Provides-Extra: native
58
- Requires-Dist: bauta-rs==0.2.1; extra == "native"
58
+ Requires-Dist: bauta-rs==0.2.2; extra == "native"
59
59
  Provides-Extra: dev
60
60
  Requires-Dist: pytest==9.1.1; extra == "dev"
61
61
  Requires-Dist: mypy==2.3.1; extra == "dev"
@@ -107,7 +107,7 @@ pip install "bauta[postgresql,oracle]"
107
107
  | `native` | `bauta-rs`, the native masker (below) | nothing on Linux (x86-64, ARM) or macOS; elsewhere, [Rust](https://rustup.rs) 1.83 or newer |
108
108
  | `all` | every driver above | nothing |
109
109
 
110
- **The native masker (optional).** `bauta-rs` masks in Rust: about ten times the throughput on one core, with identical masks. It can also mask on several cores: `jobs.yaml`'s `maskingThreads` is `1` by default, a number up to the cores available, or `auto` to divide the cores between the jobs running (see [masking threads](docs/masking.md#masking-threads)). `pip install "bauta[postgresql,native]"` installs the version that matches, which is the only one Bauta uses. Without it, everything works, only slower. See [the native masker](docs/masking.md#the-native-masker).
110
+ **The native masker (optional).** `bauta-rs` masks in Rust: about ten times the throughput on one core, with identical masks. It can also mask on several cores: `jobs.yaml`'s `maskingThreads` is `1` by default, a number up to the cores available, or `auto` to divide half the cores between the jobs running (see [masking threads](docs/masking.md#masking-threads)). `pip install "bauta[postgresql,native]"` installs the version that matches, which is the only one Bauta uses. Without it, everything works, only slower. See [the native masker](docs/masking.md#the-native-masker).
111
111
 
112
112
 
113
113
  ## Quickstart
@@ -38,7 +38,7 @@ pymssql>=2.3.5
38
38
  mysql-connector-python>=8.4
39
39
 
40
40
  [native]
41
- bauta-rs==0.2.1
41
+ bauta-rs==0.2.2
42
42
 
43
43
  [oracle]
44
44
  oracledb>=2.5
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "bauta"
7
- version = "0.2.1"
7
+ version = "0.2.2"
8
8
  description = "Safe, realistic copies of production across databases: masking, subsets, synthetic data, and the ETL to move them"
9
9
  readme = "README.md"
10
10
  license = "MIT"
@@ -78,7 +78,7 @@ all = ["bauta[mysql,postgresql,oracle,mssql,mariadb,sqlite,duckdb,files,iceberg,
78
78
  # The native masker, pinned to this exact version rather than a range: the two
79
79
  # must compute identical masks, and masking.py ignores any other version. Left
80
80
  # out of `all`, which would otherwise need Rust wherever no wheel is published.
81
- native = ["bauta-rs==0.2.1"]
81
+ native = ["bauta-rs==0.2.2"]
82
82
  dev = ["pytest==9.1.1", "mypy==2.3.1", "ruff==0.16.8", "pytest-cov==7.1.0", "types-PyYAML==6.0.12.20260906", "build==1.6.1", "twine==7.0.0"]
83
83
 
84
84
  [tool.setuptools.packages.find]
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes