bauta 0.2.2__tar.gz → 0.2.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. {bauta-0.2.2 → bauta-0.2.3}/PKG-INFO +25 -122
  2. bauta-0.2.3/README.md +132 -0
  3. {bauta-0.2.2 → bauta-0.2.3}/bauta/__init__.py +4 -3
  4. bauta-0.2.3/bauta/__main__.py +8 -0
  5. {bauta-0.2.2 → bauta-0.2.3}/bauta/cli/__init__.py +23 -8
  6. {bauta-0.2.2 → bauta-0.2.3}/bauta/cli/common.py +16 -6
  7. {bauta-0.2.2 → bauta-0.2.3}/bauta/cli/generate.py +119 -19
  8. {bauta-0.2.2 → bauta-0.2.3}/bauta/cli/review.py +71 -16
  9. {bauta-0.2.2 → bauta-0.2.3}/bauta/cli/run.py +24 -4
  10. {bauta-0.2.2 → bauta-0.2.3}/bauta/configuration/__init__.py +8 -1
  11. {bauta-0.2.2 → bauta-0.2.3}/bauta/configuration/connections.py +38 -2
  12. bauta-0.2.3/bauta/configuration/files.py +138 -0
  13. {bauta-0.2.2 → bauta-0.2.3}/bauta/configuration/models.py +33 -6
  14. {bauta-0.2.2 → bauta-0.2.3}/bauta/database/connection.py +56 -7
  15. {bauta-0.2.2 → bauta-0.2.3}/bauta/database/dialects/base.py +60 -8
  16. {bauta-0.2.2 → bauta-0.2.3}/bauta/database/dialects/duckdb.py +22 -9
  17. {bauta-0.2.2 → bauta-0.2.3}/bauta/database/dialects/mssql.py +13 -3
  18. {bauta-0.2.2 → bauta-0.2.3}/bauta/database/dialects/mysql.py +9 -5
  19. {bauta-0.2.2 → bauta-0.2.3}/bauta/database/dialects/names.py +1 -1
  20. {bauta-0.2.2 → bauta-0.2.3}/bauta/database/dialects/oracle.py +16 -7
  21. {bauta-0.2.2 → bauta-0.2.3}/bauta/database/dialects/postgresql.py +10 -5
  22. {bauta-0.2.2 → bauta-0.2.3}/bauta/database/dialects/sqlite.py +59 -10
  23. {bauta-0.2.2 → bauta-0.2.3}/bauta/generate/builtinDiscovery.py +3 -0
  24. {bauta-0.2.2 → bauta-0.2.3}/bauta/generate/discovery.py +106 -14
  25. bauta-0.2.3/bauta/generate/drift.py +233 -0
  26. {bauta-0.2.2 → bauta-0.2.3}/bauta/generate/schema.py +2 -2
  27. {bauta-0.2.2 → bauta-0.2.3}/bauta/generate/synthesize.py +5 -3
  28. bauta-0.2.3/bauta/jobs/fullRefresh.py +77 -0
  29. {bauta-0.2.2 → bauta-0.2.3}/bauta/jobs/keys.py +29 -8
  30. {bauta-0.2.2 → bauta-0.2.3}/bauta/jobs/memory.py +49 -15
  31. {bauta-0.2.2 → bauta-0.2.3}/bauta/jobs/pipeline.py +19 -9
  32. {bauta-0.2.2 → bauta-0.2.3}/bauta/jobs/reporting.py +2 -1
  33. {bauta-0.2.2 → bauta-0.2.3}/bauta/jobs/runner.py +3 -3
  34. {bauta-0.2.2 → bauta-0.2.3}/bauta/jobs/targets.py +1 -1
  35. {bauta-0.2.2 → bauta-0.2.3}/bauta/jobs/workers.py +2 -2
  36. {bauta-0.2.2 → bauta-0.2.3}/bauta/lake/__init__.py +1 -1
  37. {bauta-0.2.2 → bauta-0.2.3}/bauta/lake/iceberg.py +1 -1
  38. {bauta-0.2.2 → bauta-0.2.3}/bauta/lake/target.py +1 -1
  39. {bauta-0.2.2 → bauta-0.2.3}/bauta/log/__init__.py +2 -1
  40. {bauta-0.2.2 → bauta-0.2.3}/bauta/log/core.py +54 -1
  41. {bauta-0.2.2 → bauta-0.2.3}/bauta/log/scrubbing.py +1 -1
  42. {bauta-0.2.2 → bauta-0.2.3}/bauta/masking/__init__.py +2 -2
  43. {bauta-0.2.2 → bauta-0.2.3}/bauta/masking/core.py +10 -1
  44. {bauta-0.2.2 → bauta-0.2.3}/bauta/masking/strategies.py +176 -8
  45. {bauta-0.2.2 → bauta-0.2.3}/bauta/review/audit.py +64 -3
  46. {bauta-0.2.2 → bauta-0.2.3}/bauta/review/references.py +60 -23
  47. {bauta-0.2.2 → bauta-0.2.3}/bauta.egg-info/PKG-INFO +25 -122
  48. {bauta-0.2.2 → bauta-0.2.3}/bauta.egg-info/SOURCES.txt +4 -0
  49. {bauta-0.2.2 → bauta-0.2.3}/bauta.egg-info/requires.txt +7 -2
  50. {bauta-0.2.2 → bauta-0.2.3}/pyproject.toml +7 -4
  51. bauta-0.2.2/README.md +0 -233
  52. {bauta-0.2.2 → bauta-0.2.3}/LICENSE +0 -0
  53. {bauta-0.2.2 → bauta-0.2.3}/bauta/configuration/columnTypes.py +0 -0
  54. {bauta-0.2.2 → bauta-0.2.3}/bauta/configuration/environment.py +0 -0
  55. {bauta-0.2.2 → bauta-0.2.3}/bauta/database/__init__.py +0 -0
  56. {bauta-0.2.2 → bauta-0.2.3}/bauta/database/dialects/__init__.py +0 -0
  57. {bauta-0.2.2 → bauta-0.2.3}/bauta/database/driver.py +0 -0
  58. {bauta-0.2.2 → bauta-0.2.3}/bauta/database/values.py +0 -0
  59. {bauta-0.2.2 → bauta-0.2.3}/bauta/generate/__init__.py +0 -0
  60. {bauta-0.2.2 → bauta-0.2.3}/bauta/generate/subset.py +0 -0
  61. {bauta-0.2.2 → bauta-0.2.3}/bauta/jobs/__init__.py +0 -0
  62. {bauta-0.2.2 → bauta-0.2.3}/bauta/jobs/dependencyGraph.py +0 -0
  63. {bauta-0.2.2 → bauta-0.2.3}/bauta/lake/columnar.py +0 -0
  64. {bauta-0.2.2 → bauta-0.2.3}/bauta/lake/columns.py +0 -0
  65. {bauta-0.2.2 → bauta-0.2.3}/bauta/lake/formats.py +0 -0
  66. {bauta-0.2.2 → bauta-0.2.3}/bauta/lake/stores.py +0 -0
  67. {bauta-0.2.2 → bauta-0.2.3}/bauta/masking/fakeData.py +0 -0
  68. {bauta-0.2.2 → bauta-0.2.3}/bauta/masking/fpe.py +0 -0
  69. {bauta-0.2.2 → bauta-0.2.3}/bauta/py.typed +0 -0
  70. {bauta-0.2.2 → bauta-0.2.3}/bauta/review/__init__.py +0 -0
  71. {bauta-0.2.2 → bauta-0.2.3}/bauta/review/coverage.py +0 -0
  72. {bauta-0.2.2 → bauta-0.2.3}/bauta/transform/__init__.py +0 -0
  73. {bauta-0.2.2 → bauta-0.2.3}/bauta/transform/builtinTransforms.py +0 -0
  74. {bauta-0.2.2 → bauta-0.2.3}/bauta/transform/core.py +0 -0
  75. {bauta-0.2.2 → bauta-0.2.3}/bauta.egg-info/dependency_links.txt +0 -0
  76. {bauta-0.2.2 → bauta-0.2.3}/bauta.egg-info/entry_points.txt +0 -0
  77. {bauta-0.2.2 → bauta-0.2.3}/bauta.egg-info/top_level.txt +0 -0
  78. {bauta-0.2.2 → bauta-0.2.3}/setup.cfg +0 -0
@@ -1,11 +1,11 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: bauta
3
- Version: 0.2.2
3
+ Version: 0.2.3
4
4
  Summary: Safe, realistic copies of production across databases: masking, subsets, synthetic data, and the ETL to move them
5
5
  Author: David Ribeiro
6
6
  License-Expression: MIT
7
7
  Project-URL: Homepage, https://github.com/ribeiro11075/bauta
8
- Project-URL: Documentation, https://github.com/ribeiro11075/bauta/tree/master/docs
8
+ Project-URL: Documentation, https://ribeiro11075.github.io/bauta/
9
9
  Project-URL: Issues, https://github.com/ribeiro11075/bauta/issues
10
10
  Keywords: data masking,test data,anonymization,pseudonymization,database subsetting,synthetic data,etl,format-preserving encryption,oracle,sql server,postgresql,mysql,duckdb
11
11
  Classifier: Development Status :: 4 - Beta
@@ -55,11 +55,15 @@ Requires-Dist: cryptography>=42; extra == "fpe"
55
55
  Provides-Extra: all
56
56
  Requires-Dist: bauta[duckdb,files,fpe,iceberg,mariadb,mssql,mysql,oracle,postgresql,sqlite]; extra == "all"
57
57
  Provides-Extra: native
58
- Requires-Dist: bauta-rs==0.2.2; extra == "native"
58
+ Requires-Dist: bauta-rs==0.2.3; extra == "native"
59
+ Provides-Extra: docs
60
+ Requires-Dist: mkdocs==1.6.1; extra == "docs"
61
+ Requires-Dist: mkdocs-material==9.7.7; extra == "docs"
62
+ Requires-Dist: mike==2.2.0; extra == "docs"
59
63
  Provides-Extra: dev
60
64
  Requires-Dist: pytest==9.1.1; extra == "dev"
61
65
  Requires-Dist: mypy==2.3.1; extra == "dev"
62
- Requires-Dist: ruff==0.16.8; extra == "dev"
66
+ Requires-Dist: ruff==0.16.9; extra == "dev"
63
67
  Requires-Dist: pytest-cov==7.1.0; extra == "dev"
64
68
  Requires-Dist: types-PyYAML==6.0.12.20260906; extra == "dev"
65
69
  Requires-Dist: build==1.6.1; extra == "dev"
@@ -85,6 +89,19 @@ Dynamic: license-file
85
89
  - **No infrastructure:** a `pip install`, some YAML, and a command you run from cron.
86
90
 
87
91
 
92
+ ## Documentation
93
+
94
+ **[ribeiro11075.github.io/bauta](https://ribeiro11075.github.io/bauta/)**, for the release you have installed, or read it here in [`docs/`](docs):
95
+
96
+ | Start with | For |
97
+ | --- | --- |
98
+ | [Install](docs/get-started/install.md) and [Quickstart](docs/get-started/quickstart.md) | the drivers, the native masker, and a first run |
99
+ | [Guides](docs/guides/mask-a-table.md) | one task each: masking a table, keeping joins, subsets, proving the copy safe, running it on a schedule |
100
+ | [Reference](docs/reference/configuration.md) | every file, field, strategy and [command](docs/reference/commands/index.md) |
101
+ | [How it works](docs/concepts/how-it-works.md) and the [security model](docs/concepts/security.md) | the design, and what masking protects and what it doesn't |
102
+ | [Changelog](CHANGELOG.md) | what changed in each release, breaking changes first |
103
+
104
+
88
105
  ## Install
89
106
 
90
107
  Python 3.10 or newer. Choose the drivers you need as extras; each is loaded only when a connection uses it.
@@ -93,26 +110,12 @@ Python 3.10 or newer. Choose the drivers you need as extras; each is loaded only
93
110
  pip install "bauta[postgresql,oracle]"
94
111
  ```
95
112
 
96
- | Extra | Installs | Needs besides pip |
97
- | --- | --- | --- |
98
- | `mysql`, `mariadb` | mysql-connector-python | nothing |
99
- | `postgresql` | psycopg 3, with its own libpq | nothing |
100
- | `oracle` | oracledb, in thin mode | nothing — no Oracle client |
101
- | `mssql` | pymssql | nothing |
102
- | `sqlite` | Python's own `sqlite3` | nothing |
103
- | `duckdb` | duckdb, with pyarrow for fast loads | nothing; one process at a time per file, see [DuckDB](docs/configuration.md#duckdb) |
104
- | `files` | pyarrow, to write Parquet, CSV and JSON Lines, locally or to S3, GCS or Azure | nothing; see [files](docs/configuration.md#files) |
105
- | `iceberg` | pyiceberg, with its Glue and SQL catalogs | nothing; see [Iceberg](docs/configuration.md#iceberg) |
106
- | `fpe` | cryptography, for the `fpe` masking strategy | nothing; `oracle` already brings it |
107
- | `native` | `bauta-rs`, the native masker (below) | nothing on Linux (x86-64, ARM) or macOS; elsewhere, [Rust](https://rustup.rs) 1.83 or newer |
108
- | `all` | every driver above | nothing |
109
-
110
- **The native masker (optional).** `bauta-rs` masks in Rust: about ten times the throughput on one core, with identical masks. It can also mask on several cores: `jobs.yaml`'s `maskingThreads` is `1` by default, a number up to the cores available, or `auto` to divide half the cores between the jobs running (see [masking threads](docs/masking.md#masking-threads)). `pip install "bauta[postgresql,native]"` installs the version that matches, which is the only one Bauta uses. Without it, everything works, only slower. See [the native masker](docs/masking.md#the-native-masker).
113
+ [Install](docs/get-started/install.md) lists every extra, and the optional native masker, which masks about ten times as fast.
111
114
 
112
115
 
113
116
  ## Quickstart
114
117
 
115
- Start from the configuration in [`example/starter/configuration/`](example/starter/configuration/), from a clone or downloaded from GitHub:
118
+ Start from the configuration in [`example/starter/configuration/`](example/starter/configuration), from a clone or downloaded from GitHub:
116
119
 
117
120
  ```
118
121
  mkdir configuration
@@ -173,108 +176,8 @@ bauta --version print the version, and which masker it would use
173
176
 
174
177
  `run` makes one pass and exits, so it fits under cron or a Kubernetes CronJob. A second `run` sharing the same run state refuses to start while the first is still going. `bauta <command> --help` lists every flag.
175
178
 
176
- ### Configuration and logging
177
-
178
- Every command takes these.
179
-
180
- | Flag | Default | Effect |
181
- | --- | --- | --- |
182
- | `--config DIR` | `$BAUTA_CONFIG`, else `./configuration` | Read `jobs.yaml` and `connections.yaml` from this directory. |
183
- | `--log FILE` | none | Also write logs to this file. |
184
- | `--log-format json` | `text` | Write one JSON object per log line, for a collector. |
185
- | `--quiet` | off | Don't log to stderr. |
186
-
187
- ### Running jobs
188
-
189
- | Flag | Default | Effect |
190
- | --- | --- | --- |
191
- | `--job NAME` | every job | Run only this job, ignoring its `refresh` window. Its predecessors don't run; `run` warns about each. Repeatable. |
192
- | `--force` | off | Ignore every job's `refresh` window. |
193
- | `--forever` | off | Keep running cycles instead of exiting after one. For freshness under cron's one-minute floor. |
194
- | `--dry-run` | off | Check connections, target tables, primary keys and masking coverage, moving no rows. |
195
- | `--accept-key-change` | off | Run upsert jobs whose masking key changed since their last run. Refused where the policy masks the target's primary key; see [rotating the masking key](docs/operations.md#rotating-the-masking-key). |
196
- | `--notify-url URL` | `$BAUTA_NOTIFY_URL` | Post a JSON summary to this webhook when a cycle doesn't succeed. |
197
-
198
- ### Run state, history and the manifest
199
-
200
- Each is kept in a file or in a database table, set in [`jobs.yaml`](docs/configuration.md#file-level) or overridden for one run by its flags. A file set in `jobs.yaml` is relative to `jobs.yaml`; a file flag is relative to the working directory.
201
-
202
- | What | `jobs.yaml` setting | File | Table | Without either |
203
- | --- | --- | --- | --- | --- |
204
- | **Run state**: last runs, watermarks and key fingerprints | `memory` | `--memory FILE` | `--memory-connection ALIAS`, `--memory-table NAME` | `memory.yaml` beside `jobs.yaml` |
205
- | **History**: one record per job per cycle, for `bauta history` | `history` | `--history FILE` | `--history-connection ALIAS`, `--history-table NAME` | Not recorded. |
206
- | **Masking manifest**: what was masked and how, sealed, for `bauta verify-manifest` | `manifest` | `--manifest FILE` | `--manifest-connection ALIAS`, `--manifest-table NAME` | Not written. |
207
-
208
- Tables default to `bauta_memory`, `bauta_history` and `bauta_manifest`, and must exist first; [operations.md](docs/operations.md#tables) has their definitions. A manifest is signed when `$BAUTA_MANIFEST_KEY` is set.
209
-
210
- ### Saying less, and saying it once
211
-
212
- A `defaults:` block in [`jobs.yaml`](docs/configuration.md#defaults) supplies what every job would otherwise repeat, so a job says only what is particular to it:
213
-
214
- ```yaml
215
- defaults:
216
- sourceConnection: prod
217
- targetConnection: staging
218
- insertStrategy: upsert
219
- masking:
220
- key: ${MASKING_KEY}
221
179
 
222
- jobs:
223
- maskCustomers:
224
- sourceQuery: select id, email from customers
225
- targetTableFinal: customers
226
- masking:
227
- columns: {id: keep, email: email}
228
- ```
229
-
230
- A masking policy's `columns` stays with its job, so what a job does to its data can be read in one place. Any field a file doesn't recognise is an error, so a misspelling stops `bauta validate` rather than being quietly ignored.
231
-
232
- ### Proving the copy is safe
233
-
234
- | What | Where |
235
- | --- | --- |
236
- | Every column of every job is covered, and references still match once masked | `bauta audit --connect --strict` |
237
- | Every table in production is copied, declared, or fails the build | `bauta coverage` |
238
- | Nothing can ever reach this database unmasked | [`requireMasking`](docs/configuration.md#requiring-masking) on the alias |
239
- | This job copies as it stands, and somebody decided so | [`unmasked: true`](docs/configuration.md#copying-without-masking) on the job |
240
- | This table is deliberately not copied, and why | [`acknowledged`](docs/configuration.md#acknowledged) in `jobs.yaml` |
241
- | What was masked, how, and under which key, sealed | `bauta verify-manifest` |
242
- | No row in the copy points at a row that isn't there | `bauta verify-references` |
243
-
244
- ### Proposing and reviewing policies
245
-
246
- `discover`, `subset --mask`, `audit` and `synthesize` recognise personal data by built-in rules, and by rules of your own:
247
-
248
- | Flag | Default | Effect |
249
- | --- | --- | --- |
250
- | `--rules FILE` | `discovery.yaml` in the configuration directory, if there is one | Check these rules before the built-in ones. See [your own rules](docs/masking.md#your-own-rules-discoveryyaml). |
251
-
252
- `discover` takes `--all-tables` for a whole database instead of a repeated `--table`, and `discover`/`subset` take `--mask-keys` to mask numeric surrogate keys as well as text ones, in the domain each foreign key already shares.
253
-
254
- ### Environment variables
255
-
256
- The ones you'd set in a deployment; [operations.md](docs/operations.md#environment-variables) also lists two for diagnosis.
257
-
258
- | Variable | Effect |
259
- | --- | --- |
260
- | `BAUTA_CONFIG` | The configuration directory, when `--config` isn't given. |
261
- | `BAUTA_NOTIFY_URL` | The webhook, when `--notify-url` isn't given. |
262
- | `BAUTA_MANIFEST_KEY` | Sign manifests, and verify their signatures. |
263
- | `BAUTA_MASKING_THREADS` | Threads the native masker uses per job: a number or `auto`. Overrides `jobs.yaml`'s `maskingThreads`; see [masking threads](docs/masking.md#masking-threads). |
264
-
265
-
266
- ## Documentation
267
-
268
- | Document | Covers |
269
- | --- | --- |
270
- | [Configuration](docs/configuration.md) | every field, how credentials are read from the environment, and connection options such as TLS |
271
- | [Masking](docs/masking.md) | strategies, consistent masks across tables, the key, the manifest, `audit`, `discover`, `subset`, `schema`, `synthesize` and `clear` |
272
- | [How it works](docs/design.md) | streaming, incremental loads, retries, scheduling, and the masking design |
273
- | [Operating it](docs/operations.md) | run state, history and notifications |
274
- | [Security model](docs/security.md) | what masking protects and what it doesn't, the constructions, keys, and a deployment checklist |
275
- | [Library](docs/library.md) | embedding it in Python, results, memory backends |
276
- | [Development](docs/development.md) | running the tests, including against real databases; see also [contributing](CONTRIBUTING.md) |
277
- | [Changelog](CHANGELOG.md) | what changed in each release, breaking changes first |
180
+ Every command and flag is on its own page under [commands](docs/reference/commands/index.md).
278
181
 
279
182
 
280
183
  ## Layout
@@ -293,7 +196,7 @@ The ones you'd set in a deployment; [operations.md](docs/operations.md#environme
293
196
  | `mask-rs/` | the optional native masker, in Rust — see [its README](mask-rs/README.md) |
294
197
  | `example/` | runnable demos, each with its `configuration/`, and a starter configuration — see [its README](example/README.md) |
295
198
  | `docs/` | the documentation above |
296
- | `tests/` | the test suite, laid out like the package; see [where the tests are](docs/development.md#where-the-tests-are) |
199
+ | `tests/` | the test suite, laid out like the package; see [where the tests are](docs/project/development.md#where-the-tests-are) |
297
200
 
298
201
 
299
202
  ## License
bauta-0.2.3/README.md ADDED
@@ -0,0 +1,132 @@
1
+ # Bauta
2
+
3
+ **Put a mask on production:** a safe, realistic stand-in for your production data, in whichever database you need it. Bauta masks what it copies, copies only the slice you need with every relationship intact, generates what may not be copied at all, and moves data between databases on a schedule you already run, with nothing to host.
4
+
5
+ - **Masking:** consistent across tables and runs, one-to-one for keys (NIST FF1 where policy requires it), applied before anything reaches the target, and every column must be covered.
6
+ - **Discovery, subsets and synthetic data:** propose a masking policy from a live schema, copy a referentially complete slice of production, create the copy's tables in whichever database it goes to, and fill tables that can't be copied with generated rows.
7
+ - **Audit:** report what every job does with data and what a reviewer should question, and seal each run's masking manifest so it can be verified later.
8
+ - **Provable coverage:** list every table in production and what the jobs do with each, so a table nobody wrote a job for fails the build rather than going unnoticed. A database can require that nothing reaches it unmasked.
9
+ - **Seven databases:** Oracle, SQL Server, PostgreSQL, MySQL, MariaDB, SQLite and DuckDB, as source or target in any combination.
10
+ - **Files as a target:** a masked copy in Parquet, CSV or JSON Lines, in a directory or in S3, Google Cloud Storage or Azure, for a data lake that Athena, Snowflake or Databricks reads: appended to incrementally or published whole as a snapshot, with nothing visible until a run succeeds.
11
+ - **Iceberg tables:** a masked copy as Iceberg tables in any of the three clouds, through a Glue, REST or SQL catalog: each run one commit, upserts that merge by key, and old snapshots' files deleted with them.
12
+ - **Streaming:** memory stays flat however large the table, and PostgreSQL and SQL Server targets load in bulk.
13
+ - **Fast:** a million rows of six masked columns, two of them one-to-one keys, in under 8 seconds on one core with the optional native masker, and 74 without; more cores for wide tables. Either way the masks are the same.
14
+ - **Incremental loads:** extract only what changed since the last successful run.
15
+ - **A dependency graph:** jobs run in order, concurrently where they can, each in its own process with an optional timeout.
16
+ - **Operable:** webhook alerts; run state, history and manifests each in a file or a table; and passwords from a command for cloud IAM tokens.
17
+ - **No infrastructure:** a `pip install`, some YAML, and a command you run from cron.
18
+
19
+
20
+ ## Documentation
21
+
22
+ **[ribeiro11075.github.io/bauta](https://ribeiro11075.github.io/bauta/)**, for the release you have installed, or read it here in [`docs/`](docs):
23
+
24
+ | Start with | For |
25
+ | --- | --- |
26
+ | [Install](docs/get-started/install.md) and [Quickstart](docs/get-started/quickstart.md) | the drivers, the native masker, and a first run |
27
+ | [Guides](docs/guides/mask-a-table.md) | one task each: masking a table, keeping joins, subsets, proving the copy safe, running it on a schedule |
28
+ | [Reference](docs/reference/configuration.md) | every file, field, strategy and [command](docs/reference/commands/index.md) |
29
+ | [How it works](docs/concepts/how-it-works.md) and the [security model](docs/concepts/security.md) | the design, and what masking protects and what it doesn't |
30
+ | [Changelog](CHANGELOG.md) | what changed in each release, breaking changes first |
31
+
32
+
33
+ ## Install
34
+
35
+ Python 3.10 or newer. Choose the drivers you need as extras; each is loaded only when a connection uses it.
36
+
37
+ ```
38
+ pip install "bauta[postgresql,oracle]"
39
+ ```
40
+
41
+ [Install](docs/get-started/install.md) lists every extra, and the optional native masker, which masks about ten times as fast.
42
+
43
+
44
+ ## Quickstart
45
+
46
+ Start from the configuration in [`example/starter/configuration/`](example/starter/configuration), from a clone or downloaded from GitHub:
47
+
48
+ ```
49
+ mkdir configuration
50
+ cp example/starter/configuration/*.yaml configuration/
51
+ ```
52
+
53
+ Edit `configuration/connections.yaml` and `configuration/jobs.yaml` for your databases, then supply the credentials they reference:
54
+
55
+ ```
56
+ export SOURCE_DB_PASSWORD=... TARGET_DB_PASSWORD=... MASKING_KEY=...
57
+
58
+ bauta validate # check the configuration, offline
59
+ bauta run --dry-run # check connections and tables, moving nothing
60
+ bauta run # run every job once
61
+ ```
62
+
63
+ To see it work without any of that, the demos in a clone of this repository use throwaway SQLite databases:
64
+
65
+ ```
66
+ git clone https://github.com/ribeiro11075/bauta.git && cd bauta
67
+ pip install -e ".[fpe]"
68
+
69
+ python example/walkthrough/demo.py # the whole workflow: discover, subset, audit, mask, verify, synthesize
70
+ python example/incremental/demo.py # streaming and incremental loads
71
+ python example/masking/demo.py # masking, discovery and a subset, from Python
72
+ python example/native-masking/demo.py # Python against Rust, and Rust on one core against all of them
73
+ ```
74
+
75
+
76
+ ## The command
77
+
78
+ ```
79
+ bauta run run every job that's due, once
80
+ bauta validate check the configuration, without connecting
81
+ bauta jobs show the job graph and which jobs are due
82
+ bauta history show recent job outcomes
83
+
84
+ bauta discover propose a masking policy for tables
85
+ bauta subset generate jobs that copy a referentially complete subset
86
+ bauta schema create target tables from source ones, in the target's dialect
87
+ bauta synthesize fill tables with generated rows, for data that can't be copied
88
+ bauta clear empty the jobs' target tables, children first
89
+
90
+ bauta audit report what each job does with data, and what to question
91
+ bauta coverage list a source database's tables and what the jobs do with each
92
+ bauta verify-manifest check a masking manifest is unaltered, and who signed it
93
+ bauta verify-references count rows in the copy whose foreign key points at nothing
94
+
95
+ bauta --version print the version, and which masker it would use
96
+ ```
97
+
98
+ | Exit code | Meaning |
99
+ | --- | --- |
100
+ | `0` | Every job completed. |
101
+ | `1` | A job failed, or was skipped because a predecessor failed, or the command failed on a database error. |
102
+ | `2` | Invalid configuration or usage. |
103
+ | `130` | Interrupted by a signal: running jobs finished, the rest were skipped. |
104
+
105
+ `run` makes one pass and exits, so it fits under cron or a Kubernetes CronJob. A second `run` sharing the same run state refuses to start while the first is still going. `bauta <command> --help` lists every flag.
106
+
107
+
108
+ Every command and flag is on its own page under [commands](docs/reference/commands/index.md).
109
+
110
+
111
+ ## Layout
112
+
113
+ | Path | What it is |
114
+ | --- | --- |
115
+ | `bauta/configuration/` | reading and validating the YAML: `${NAME}` and `passwordCommand` in `environment.py`, the models in `models.py` |
116
+ | `bauta/database/` | streaming and loading rows in `connection.py`; what differs between the seven databases in `dialects/`, one module each |
117
+ | `bauta/jobs/` | `runner.py` runs a cycle of jobs, each in a process of its own (`workers.py`) moving rows a chunk at a time (`pipeline.py`); run state in `memory.py`, history and manifests in `reporting.py` |
118
+ | `bauta/masking/` | the keyed hash, `Strategy` and masking plans in `core.py`, the built-in strategies in `strategies.py` |
119
+ | `bauta/transform/` | per-column transforms, applied before masking, and the ones that ship in `builtinTransforms.py` |
120
+ | `bauta/generate/` | `discover`, `subset`, `schema` and `synthesize`: what is built from a live schema |
121
+ | `bauta/review/` | `audit`, `coverage` and `verify-references`: reports that move no data |
122
+ | `bauta/log/` | logging, and the scrubbing that keeps values out of every message |
123
+ | `bauta/cli/` | the `bauta` command, one module per group of subcommands |
124
+ | `mask-rs/` | the optional native masker, in Rust — see [its README](mask-rs/README.md) |
125
+ | `example/` | runnable demos, each with its `configuration/`, and a starter configuration — see [its README](example/README.md) |
126
+ | `docs/` | the documentation above |
127
+ | `tests/` | the test suite, laid out like the package; see [where the tests are](docs/project/development.md#where-the-tests-are) |
128
+
129
+
130
+ ## License
131
+
132
+ [MIT](LICENSE)
@@ -1,13 +1,13 @@
1
1
  """Safe, realistic copies of production across databases. Most deployments
2
- use the `bauta` command; see docs/library.md for embedding it.
2
+ use the `bauta` command; see docs/reference/python-api.md for embedding it.
3
3
 
4
- What is importable from here is the public API, the one docs/library.md
4
+ What is importable from here is the public API, the one docs/reference/python-api.md
5
5
  describes. A name reached only through a submodule is internal, unless the
6
6
  documentation names it there, and may change in any release.
7
7
  """
8
8
  from .configuration import (Configuration, ConfigurationError, ConnectionConfig, DatabaseType, DataJobConfig, DataJobsFile, DiscoveryRulesFile,
9
9
  DuckDBConnection, FilesConnection, InsertStrategy, MariaDBConnection, MaskingConfig, MSSQLConnection, MySQLConnection, OracleConnection,
10
- PostgreSQLConnection, SQLiteConnection, connectionConfig, expandEnvironmentVariables)
10
+ PostgreSQLConnection, SQLiteConnection, connectionConfig, expandEnvironmentVariables, readJobsFile)
11
11
  from .database import Database
12
12
  from .jobs.dependencyGraph import DependencyGraph, JobOutcome, JobStatus
13
13
  from .database.dialects import ForeignKey
@@ -27,6 +27,7 @@ __all__ = [
27
27
  'auditJobs',
28
28
  'buildMaskingManifest',
29
29
  'Configuration',
30
+ 'readJobsFile',
30
31
  'ConfigurationError',
31
32
  'Database',
32
33
  'DATABASE_HISTORY_SCHEMA',
@@ -0,0 +1,8 @@
1
+ """`python -m bauta`: the bauta command, where its script isn't on PATH --
2
+ an orchestrator's worker, say, running the interpreter it was installed into.
3
+ """
4
+ import sys
5
+
6
+ from .cli import main
7
+
8
+ sys.exit(main())
@@ -47,7 +47,7 @@ def _addCommonArguments(parser: argparse.ArgumentParser, jobs: bool = True, memo
47
47
  if jobs and memory:
48
48
  parser.add_argument('--memory', help='path to the run-memory file (default: jobs.yaml\'s `memory`, else memory.yaml beside jobs.yaml)')
49
49
  parser.add_argument('--memory-connection', metavar='ALIAS',
50
- help='keep run memory in a table of this connection instead of a file (see docs/operations.md)')
50
+ help='keep run memory in a table of this connection instead of a file (see docs/guides/run-on-a-schedule.md)')
51
51
  parser.add_argument('--memory-table', help='the run-memory table (default: jobs.yaml\'s, else bauta_memory)')
52
52
  _addLoggingArguments(parser)
53
53
 
@@ -73,11 +73,11 @@ def _addManifestKeyArgument(parser: argparse.ArgumentParser) -> None:
73
73
 
74
74
  def _addLoggingArguments(parser: argparse.ArgumentParser) -> None:
75
75
 
76
- parser.add_argument('--log', help='also write logs to this file (logs always go to stderr unless --quiet)')
76
+ parser.add_argument('--log', help='also write logs to this file (logs always go to stderr; --quiet leaves only errors there)')
77
77
  parser.add_argument('--log-level', default='info', choices=['debug', 'info', 'warning', 'error'], help='default: info')
78
78
  parser.add_argument('--log-format', default='text', choices=['text', 'json'],
79
79
  help='json emits one object per record, carrying job/status/rowCount as fields a log collector can filter and alert on')
80
- parser.add_argument('--quiet', action='store_true', help='do not log to stderr')
80
+ parser.add_argument('--quiet', action='store_true', help='log only errors to stderr, one line each, without tracebacks')
81
81
 
82
82
 
83
83
  def _positiveInteger(text: str) -> int:
@@ -112,6 +112,9 @@ def _addGeneratorArguments(parser: argparse.ArgumentParser) -> None:
112
112
  parser.add_argument('--sample', type=_positiveInteger, default=1000, help='rows sampled per table to classify columns (default: 1000)')
113
113
  parser.add_argument('--key-variable', default='MASKING_KEY', help='environment variable the generated jobs read the masking key from')
114
114
  parser.add_argument('--chunk-size', type=_positiveInteger, default=5000, help='chunkSize for the generated jobs (default: 5000)')
115
+ parser.add_argument('--self-contained', action='store_true',
116
+ help='write the connections, insert strategy and key into every job rather than under defaults:, '
117
+ 'for jobs to paste into a file with defaults of its own')
115
118
  parser.add_argument('--mask-keys', action='store_true',
116
119
  help='mask numeric surrogate keys too, in the domain each foreign key shares, instead of keeping them')
117
120
  parser.add_argument('--output', help='write the generated jobs here instead of stdout; must not already exist')
@@ -154,6 +157,9 @@ def _buildParser() -> argparse.ArgumentParser:
154
157
  help='write a JSON record of what was masked, how, and under which key fingerprint (default: jobs.yaml\'s `manifest`)')
155
158
  _addManifestLocationArguments(runParser)
156
159
  _addManifestKeyArgument(runParser)
160
+ runParser.add_argument('--full-refresh', action='store_true',
161
+ help='load every incremental job\'s whole source and replace its target, so rows deleted from the source '
162
+ 'leave the copy; implies --force. Database targets need targetTableStage')
157
163
  runParser.add_argument('--accept-key-change', action='store_true',
158
164
  help='run upsert jobs even though their masking key changed since their last run')
159
165
  _addHistoryArguments(runParser)
@@ -216,14 +222,21 @@ def _buildParser() -> argparse.ArgumentParser:
216
222
  jobsParser.set_defaults(handler=_commandJobs)
217
223
 
218
224
  discoverParser = subparsers.add_parser('discover', help='propose a masking policy for tables, from their schema and a sample')
219
- _addCommonArguments(discoverParser, jobs=False)
220
- discoverParser.add_argument('--connection', required=True, help='the alias to read from')
225
+ _addCommonArguments(discoverParser, memory=False)
226
+ discoverParser.add_argument('--connection', help='the alias to read from; required unless --update')
221
227
  discoverParser.add_argument('--table', action='append', help='a table to propose a policy for (repeatable)')
222
228
  discoverParser.add_argument('--all-tables', action='store_true', help='every table in the database, instead of naming each with --table')
223
229
  discoverParser.add_argument('--schema', help='the schema --all-tables lists, instead of the connection\'s own')
224
- discoverParser.add_argument('--target', help='the alias the generated jobs load into (default: --database, masking in place)')
230
+ discoverParser.add_argument('--target', help='the alias the generated jobs load into (default: --connection, masking in place)')
225
231
  _addGeneratorArguments(discoverParser)
226
232
  _addRulesArgument(discoverParser)
233
+ discoverParser.add_argument('--update', action='store_true',
234
+ help='compare each masked job in the jobs file with what its sourceQuery returns now, and propose a policy '
235
+ 'for each new column; exits 1 if any job has drifted')
236
+ discoverParser.add_argument('--job', action='append', help='with --update, only this job (repeatable)')
237
+ discoverParser.add_argument('--apply', action='store_true',
238
+ help='with --update, write the proposals into the file each job is defined in, and drop the columns '
239
+ 'its sourceQuery no longer returns')
227
240
  discoverParser.set_defaults(handler=_commandDiscover)
228
241
 
229
242
  subsetParser = subparsers.add_parser('subset', help='generate jobs that copy a referentially complete subset')
@@ -244,11 +257,13 @@ def _buildParser() -> argparse.ArgumentParser:
244
257
  _addCommonArguments(schemaParser, jobs=False)
245
258
  schemaParser.add_argument('--connection', required=True, help='the alias to read table definitions from')
246
259
  schemaParser.add_argument('--target', required=True, help='the alias the tables are for; its dialect decides the types')
247
- schemaParser.add_argument('--table', action='append', required=True, help='a table to create (repeatable)')
260
+ schemaParser.add_argument('--table', action='append', help='a table to create (repeatable)')
261
+ schemaParser.add_argument('--all-tables', action='store_true', help='every table in the database, instead of naming each with --table')
262
+ schemaParser.add_argument('--schema', help='the schema --all-tables lists, instead of the connection\'s own')
248
263
  schemaParser.add_argument('--related', action='store_true', help='also every table a subset rooted at --table would copy')
249
264
  schemaParser.add_argument('--no-children', action='store_true', help='with --related, only the tables --table references')
250
265
  schemaParser.add_argument('--no-foreign-keys', action='store_true', help='leave foreign keys out of the generated tables')
251
- schemaParser.add_argument('--stage-suffix', help='also create <table><suffix> stage tables, for swap jobs; alone if --target is --database')
266
+ schemaParser.add_argument('--stage-suffix', help='also create <table><suffix> stage tables, for swap jobs; alone if --target is --connection')
252
267
  schemaParser.add_argument('--apply', action='store_true', help='create the tables in --target, skipping any that already exist')
253
268
  schemaParser.add_argument('--output', help='write the SQL here instead of stdout; must not already exist')
254
269
  schemaParser.set_defaults(handler=_commandSchema)
@@ -14,7 +14,7 @@ from typing import Any, Dict, Generator, List, Mapping, Optional, Sequence, Tupl
14
14
  import yaml
15
15
 
16
16
  from ..configuration import (Configuration, ConfigurationError, ConnectionConfig, DatabaseConfig, DataJobConfig, DataJobsFile, FilesConnection, IcebergConnection, isLake,
17
- StorageLocation, TableLocation, expandEnvironmentVariables)
17
+ JobsDocument, StorageLocation, TableLocation, expandEnvironmentVariables, readJobsFile)
18
18
  from ..database import Database
19
19
  from ..log import Log
20
20
  from ..masking import MaskingError
@@ -58,6 +58,12 @@ def _loadYaml(path: Path) -> Any:
58
58
  raise UsageError('{} is not valid YAML: {}'.format(path, error)) from error
59
59
 
60
60
 
61
+ def _readJobs(jobsPath: Path) -> JobsDocument:
62
+ """The jobs file with the files its `include` names merged in."""
63
+
64
+ return readJobsFile(jobsPath, load=_loadYaml)
65
+
66
+
61
67
  CONNECTIONS_FILE = 'connections.yaml'
62
68
 
63
69
 
@@ -162,12 +168,16 @@ def _history(location: Location, connectionConfiguration: Dict[str, ConnectionCo
162
168
 
163
169
 
164
170
  def _configureLogging(arguments: argparse.Namespace) -> Log:
165
- """stderr unless --quiet, where a container collects it; --log adds a file."""
171
+ """stderr, where a container collects it; --log adds a file. --quiet
172
+ leaves only errors on stderr, one line each.
173
+ """
166
174
 
167
175
  level = getattr(logging, arguments.log_level.upper())
168
176
  log = Log(logFile=arguments.log, level=level, logFormat=arguments.log_format)
169
177
 
170
- if not arguments.quiet:
178
+ if arguments.quiet:
179
+ log.addErrorStream(stream=sys.stderr)
180
+ else:
171
181
  log.addStreamHandler(stream=sys.stderr, level=level)
172
182
 
173
183
  return log
@@ -177,14 +187,14 @@ def _loadConnections(arguments: argparse.Namespace) -> Dict[str, ConnectionConfi
177
187
 
178
188
  _, connectionsPath = _resolveConfigurationPaths(arguments)
179
189
 
180
- return Configuration.validateConnectionConfiguration(_loadYaml(connectionsPath))
190
+ return Configuration.validateConnectionConfiguration(_loadYaml(connectionsPath), directory=str(connectionsPath.parent))
181
191
 
182
192
 
183
193
  def _loadDataJobs(arguments: argparse.Namespace) -> Tuple[DataJobsFile, Dict[str, ConnectionConfig]]:
184
194
 
185
195
  jobsPath, connectionsPath = _resolveConfigurationPaths(arguments)
186
- connectionConfiguration = Configuration.validateConnectionConfiguration(_loadYaml(connectionsPath))
187
- jobsFile = Configuration.validateJobConfiguration(_loadYaml(jobsPath), DataJobsFile)
196
+ connectionConfiguration = Configuration.validateConnectionConfiguration(_loadYaml(connectionsPath), directory=str(connectionsPath.parent))
197
+ jobsFile = Configuration.validateJobConfiguration(_readJobs(jobsPath).content, DataJobsFile)
188
198
  Configuration.validateJobGraph(jobsFile.jobs, connections=connectionConfiguration)
189
199
 
190
200
  unknown = ['{}: connection "{}" is not a known connection alias'.format(setting, location.connection)