bauta 0.2.2__tar.gz → 0.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {bauta-0.2.2 → bauta-0.2.4}/PKG-INFO +26 -123
- bauta-0.2.4/README.md +132 -0
- {bauta-0.2.2 → bauta-0.2.4}/bauta/__init__.py +4 -3
- bauta-0.2.4/bauta/__main__.py +8 -0
- {bauta-0.2.2 → bauta-0.2.4}/bauta/cli/__init__.py +38 -10
- {bauta-0.2.2 → bauta-0.2.4}/bauta/cli/common.py +35 -6
- {bauta-0.2.2 → bauta-0.2.4}/bauta/cli/generate.py +119 -19
- {bauta-0.2.2 → bauta-0.2.4}/bauta/cli/review.py +97 -21
- {bauta-0.2.2 → bauta-0.2.4}/bauta/cli/run.py +36 -5
- {bauta-0.2.2 → bauta-0.2.4}/bauta/configuration/__init__.py +12 -3
- {bauta-0.2.2 → bauta-0.2.4}/bauta/configuration/connections.py +38 -2
- bauta-0.2.4/bauta/configuration/files.py +138 -0
- {bauta-0.2.2 → bauta-0.2.4}/bauta/configuration/models.py +120 -11
- {bauta-0.2.2 → bauta-0.2.4}/bauta/database/connection.py +107 -7
- {bauta-0.2.2 → bauta-0.2.4}/bauta/database/dialects/base.py +142 -8
- {bauta-0.2.2 → bauta-0.2.4}/bauta/database/dialects/duckdb.py +62 -9
- {bauta-0.2.2 → bauta-0.2.4}/bauta/database/dialects/mssql.py +68 -3
- {bauta-0.2.2 → bauta-0.2.4}/bauta/database/dialects/mysql.py +20 -5
- {bauta-0.2.2 → bauta-0.2.4}/bauta/database/dialects/names.py +1 -1
- {bauta-0.2.2 → bauta-0.2.4}/bauta/database/dialects/oracle.py +30 -7
- {bauta-0.2.2 → bauta-0.2.4}/bauta/database/dialects/postgresql.py +28 -5
- {bauta-0.2.2 → bauta-0.2.4}/bauta/database/dialects/sqlite.py +117 -11
- {bauta-0.2.2 → bauta-0.2.4}/bauta/generate/builtinDiscovery.py +12 -4
- {bauta-0.2.2 → bauta-0.2.4}/bauta/generate/discovery.py +175 -22
- bauta-0.2.4/bauta/generate/drift.py +233 -0
- {bauta-0.2.2 → bauta-0.2.4}/bauta/generate/schema.py +2 -2
- {bauta-0.2.2 → bauta-0.2.4}/bauta/generate/synthesize.py +9 -5
- {bauta-0.2.2 → bauta-0.2.4}/bauta/jobs/__init__.py +2 -1
- {bauta-0.2.2 → bauta-0.2.4}/bauta/jobs/dependencyGraph.py +46 -2
- bauta-0.2.4/bauta/jobs/fullRefresh.py +77 -0
- {bauta-0.2.2 → bauta-0.2.4}/bauta/jobs/keys.py +67 -25
- {bauta-0.2.2 → bauta-0.2.4}/bauta/jobs/memory.py +49 -15
- bauta-0.2.4/bauta/jobs/partitions.py +193 -0
- bauta-0.2.4/bauta/jobs/pipeline.py +628 -0
- {bauta-0.2.2 → bauta-0.2.4}/bauta/jobs/reporting.py +2 -1
- {bauta-0.2.2 → bauta-0.2.4}/bauta/jobs/runner.py +31 -5
- {bauta-0.2.2 → bauta-0.2.4}/bauta/jobs/targets.py +71 -6
- {bauta-0.2.2 → bauta-0.2.4}/bauta/jobs/workers.py +10 -6
- {bauta-0.2.2 → bauta-0.2.4}/bauta/lake/__init__.py +1 -1
- {bauta-0.2.2 → bauta-0.2.4}/bauta/lake/iceberg.py +3 -2
- {bauta-0.2.2 → bauta-0.2.4}/bauta/lake/target.py +3 -2
- {bauta-0.2.2 → bauta-0.2.4}/bauta/log/__init__.py +5 -1
- {bauta-0.2.2 → bauta-0.2.4}/bauta/log/core.py +112 -5
- bauta-0.2.4/bauta/log/scrubbing.py +258 -0
- {bauta-0.2.2 → bauta-0.2.4}/bauta/masking/__init__.py +13 -6
- {bauta-0.2.2 → bauta-0.2.4}/bauta/masking/core.py +235 -15
- bauta-0.2.4/bauta/masking/fakeData.py +563 -0
- {bauta-0.2.2 → bauta-0.2.4}/bauta/masking/strategies.py +560 -29
- {bauta-0.2.2 → bauta-0.2.4}/bauta/review/audit.py +209 -19
- bauta-0.2.4/bauta/review/html.py +259 -0
- {bauta-0.2.2 → bauta-0.2.4}/bauta/review/references.py +60 -23
- {bauta-0.2.2 → bauta-0.2.4}/bauta.egg-info/PKG-INFO +26 -123
- {bauta-0.2.2 → bauta-0.2.4}/bauta.egg-info/SOURCES.txt +6 -0
- {bauta-0.2.2 → bauta-0.2.4}/bauta.egg-info/requires.txt +8 -3
- {bauta-0.2.2 → bauta-0.2.4}/pyproject.toml +12 -6
- bauta-0.2.2/README.md +0 -233
- bauta-0.2.2/bauta/jobs/pipeline.py +0 -362
- bauta-0.2.2/bauta/log/scrubbing.py +0 -115
- bauta-0.2.2/bauta/masking/fakeData.py +0 -162
- {bauta-0.2.2 → bauta-0.2.4}/LICENSE +0 -0
- {bauta-0.2.2 → bauta-0.2.4}/bauta/configuration/columnTypes.py +0 -0
- {bauta-0.2.2 → bauta-0.2.4}/bauta/configuration/environment.py +0 -0
- {bauta-0.2.2 → bauta-0.2.4}/bauta/database/__init__.py +0 -0
- {bauta-0.2.2 → bauta-0.2.4}/bauta/database/dialects/__init__.py +0 -0
- {bauta-0.2.2 → bauta-0.2.4}/bauta/database/driver.py +0 -0
- {bauta-0.2.2 → bauta-0.2.4}/bauta/database/values.py +0 -0
- {bauta-0.2.2 → bauta-0.2.4}/bauta/generate/__init__.py +0 -0
- {bauta-0.2.2 → bauta-0.2.4}/bauta/generate/subset.py +0 -0
- {bauta-0.2.2 → bauta-0.2.4}/bauta/lake/columnar.py +0 -0
- {bauta-0.2.2 → bauta-0.2.4}/bauta/lake/columns.py +0 -0
- {bauta-0.2.2 → bauta-0.2.4}/bauta/lake/formats.py +0 -0
- {bauta-0.2.2 → bauta-0.2.4}/bauta/lake/stores.py +0 -0
- {bauta-0.2.2 → bauta-0.2.4}/bauta/masking/fpe.py +0 -0
- {bauta-0.2.2 → bauta-0.2.4}/bauta/py.typed +0 -0
- {bauta-0.2.2 → bauta-0.2.4}/bauta/review/__init__.py +0 -0
- {bauta-0.2.2 → bauta-0.2.4}/bauta/review/coverage.py +0 -0
- {bauta-0.2.2 → bauta-0.2.4}/bauta/transform/__init__.py +0 -0
- {bauta-0.2.2 → bauta-0.2.4}/bauta/transform/builtinTransforms.py +0 -0
- {bauta-0.2.2 → bauta-0.2.4}/bauta/transform/core.py +0 -0
- {bauta-0.2.2 → bauta-0.2.4}/bauta.egg-info/dependency_links.txt +0 -0
- {bauta-0.2.2 → bauta-0.2.4}/bauta.egg-info/entry_points.txt +0 -0
- {bauta-0.2.2 → bauta-0.2.4}/bauta.egg-info/top_level.txt +0 -0
- {bauta-0.2.2 → bauta-0.2.4}/setup.cfg +0 -0
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: bauta
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.4
|
|
4
4
|
Summary: Safe, realistic copies of production across databases: masking, subsets, synthetic data, and the ETL to move them
|
|
5
5
|
Author: David Ribeiro
|
|
6
6
|
License-Expression: MIT
|
|
7
7
|
Project-URL: Homepage, https://github.com/ribeiro11075/bauta
|
|
8
|
-
Project-URL: Documentation, https://github.
|
|
8
|
+
Project-URL: Documentation, https://ribeiro11075.github.io/bauta/
|
|
9
9
|
Project-URL: Issues, https://github.com/ribeiro11075/bauta/issues
|
|
10
10
|
Keywords: data masking,test data,anonymization,pseudonymization,database subsetting,synthetic data,etl,format-preserving encryption,oracle,sql server,postgresql,mysql,duckdb
|
|
11
11
|
Classifier: Development Status :: 4 - Beta
|
|
@@ -42,7 +42,7 @@ Provides-Extra: mariadb
|
|
|
42
42
|
Requires-Dist: mysql-connector-python>=8.4; extra == "mariadb"
|
|
43
43
|
Provides-Extra: sqlite
|
|
44
44
|
Provides-Extra: duckdb
|
|
45
|
-
Requires-Dist: duckdb>=1.
|
|
45
|
+
Requires-Dist: duckdb>=1.5; extra == "duckdb"
|
|
46
46
|
Requires-Dist: pyarrow>=21; extra == "duckdb"
|
|
47
47
|
Requires-Dist: pytz>=2024.1; extra == "duckdb"
|
|
48
48
|
Provides-Extra: files
|
|
@@ -55,11 +55,15 @@ Requires-Dist: cryptography>=42; extra == "fpe"
|
|
|
55
55
|
Provides-Extra: all
|
|
56
56
|
Requires-Dist: bauta[duckdb,files,fpe,iceberg,mariadb,mssql,mysql,oracle,postgresql,sqlite]; extra == "all"
|
|
57
57
|
Provides-Extra: native
|
|
58
|
-
Requires-Dist: bauta-rs==0.2.
|
|
58
|
+
Requires-Dist: bauta-rs==0.2.4; extra == "native"
|
|
59
|
+
Provides-Extra: docs
|
|
60
|
+
Requires-Dist: mkdocs==1.6.1; extra == "docs"
|
|
61
|
+
Requires-Dist: mkdocs-material==9.7.7; extra == "docs"
|
|
62
|
+
Requires-Dist: mike==2.2.0; extra == "docs"
|
|
59
63
|
Provides-Extra: dev
|
|
60
64
|
Requires-Dist: pytest==9.1.1; extra == "dev"
|
|
61
65
|
Requires-Dist: mypy==2.3.1; extra == "dev"
|
|
62
|
-
Requires-Dist: ruff==0.16.
|
|
66
|
+
Requires-Dist: ruff==0.16.9; extra == "dev"
|
|
63
67
|
Requires-Dist: pytest-cov==7.1.0; extra == "dev"
|
|
64
68
|
Requires-Dist: types-PyYAML==6.0.12.20260906; extra == "dev"
|
|
65
69
|
Requires-Dist: build==1.6.1; extra == "dev"
|
|
@@ -85,6 +89,19 @@ Dynamic: license-file
|
|
|
85
89
|
- **No infrastructure:** a `pip install`, some YAML, and a command you run from cron.
|
|
86
90
|
|
|
87
91
|
|
|
92
|
+
## Documentation
|
|
93
|
+
|
|
94
|
+
**[ribeiro11075.github.io/bauta](https://ribeiro11075.github.io/bauta/)**, for the release you have installed, or read it here in [`docs/`](docs):
|
|
95
|
+
|
|
96
|
+
| Start with | For |
|
|
97
|
+
| --- | --- |
|
|
98
|
+
| [Install](docs/get-started/install.md) and [Quickstart](docs/get-started/quickstart.md) | the drivers, the native masker, and a first run |
|
|
99
|
+
| [Guides](docs/guides/mask-a-table.md) | one task each: masking a table, keeping joins, subsets, proving the copy safe, running it on a schedule |
|
|
100
|
+
| [Reference](docs/reference/configuration.md) | every file, field, strategy and [command](docs/reference/commands/index.md) |
|
|
101
|
+
| [How it works](docs/concepts/how-it-works.md) and the [security model](docs/concepts/security.md) | the design, and what masking protects and what it doesn't |
|
|
102
|
+
| [Changelog](CHANGELOG.md) | what changed in each release, breaking changes first |
|
|
103
|
+
|
|
104
|
+
|
|
88
105
|
## Install
|
|
89
106
|
|
|
90
107
|
Python 3.10 or newer. Choose the drivers you need as extras; each is loaded only when a connection uses it.
|
|
@@ -93,26 +110,12 @@ Python 3.10 or newer. Choose the drivers you need as extras; each is loaded only
|
|
|
93
110
|
pip install "bauta[postgresql,oracle]"
|
|
94
111
|
```
|
|
95
112
|
|
|
96
|
-
|
|
97
|
-
| --- | --- | --- |
|
|
98
|
-
| `mysql`, `mariadb` | mysql-connector-python | nothing |
|
|
99
|
-
| `postgresql` | psycopg 3, with its own libpq | nothing |
|
|
100
|
-
| `oracle` | oracledb, in thin mode | nothing — no Oracle client |
|
|
101
|
-
| `mssql` | pymssql | nothing |
|
|
102
|
-
| `sqlite` | Python's own `sqlite3` | nothing |
|
|
103
|
-
| `duckdb` | duckdb, with pyarrow for fast loads | nothing; one process at a time per file, see [DuckDB](docs/configuration.md#duckdb) |
|
|
104
|
-
| `files` | pyarrow, to write Parquet, CSV and JSON Lines, locally or to S3, GCS or Azure | nothing; see [files](docs/configuration.md#files) |
|
|
105
|
-
| `iceberg` | pyiceberg, with its Glue and SQL catalogs | nothing; see [Iceberg](docs/configuration.md#iceberg) |
|
|
106
|
-
| `fpe` | cryptography, for the `fpe` masking strategy | nothing; `oracle` already brings it |
|
|
107
|
-
| `native` | `bauta-rs`, the native masker (below) | nothing on Linux (x86-64, ARM) or macOS; elsewhere, [Rust](https://rustup.rs) 1.83 or newer |
|
|
108
|
-
| `all` | every driver above | nothing |
|
|
109
|
-
|
|
110
|
-
**The native masker (optional).** `bauta-rs` masks in Rust: about ten times the throughput on one core, with identical masks. It can also mask on several cores: `jobs.yaml`'s `maskingThreads` is `1` by default, a number up to the cores available, or `auto` to divide half the cores between the jobs running (see [masking threads](docs/masking.md#masking-threads)). `pip install "bauta[postgresql,native]"` installs the version that matches, which is the only one Bauta uses. Without it, everything works, only slower. See [the native masker](docs/masking.md#the-native-masker).
|
|
113
|
+
[Install](docs/get-started/install.md) lists every extra, and the optional native masker, which masks about ten times as fast.
|
|
111
114
|
|
|
112
115
|
|
|
113
116
|
## Quickstart
|
|
114
117
|
|
|
115
|
-
Start from the configuration in [`example/starter/configuration/`](example/starter/configuration
|
|
118
|
+
Start from the configuration in [`example/starter/configuration/`](example/starter/configuration), from a clone or downloaded from GitHub:
|
|
116
119
|
|
|
117
120
|
```
|
|
118
121
|
mkdir configuration
|
|
@@ -173,108 +176,8 @@ bauta --version print the version, and which masker it would use
|
|
|
173
176
|
|
|
174
177
|
`run` makes one pass and exits, so it fits under cron or a Kubernetes CronJob. A second `run` sharing the same run state refuses to start while the first is still going. `bauta <command> --help` lists every flag.
|
|
175
178
|
|
|
176
|
-
### Configuration and logging
|
|
177
|
-
|
|
178
|
-
Every command takes these.
|
|
179
|
-
|
|
180
|
-
| Flag | Default | Effect |
|
|
181
|
-
| --- | --- | --- |
|
|
182
|
-
| `--config DIR` | `$BAUTA_CONFIG`, else `./configuration` | Read `jobs.yaml` and `connections.yaml` from this directory. |
|
|
183
|
-
| `--log FILE` | none | Also write logs to this file. |
|
|
184
|
-
| `--log-format json` | `text` | Write one JSON object per log line, for a collector. |
|
|
185
|
-
| `--quiet` | off | Don't log to stderr. |
|
|
186
|
-
|
|
187
|
-
### Running jobs
|
|
188
|
-
|
|
189
|
-
| Flag | Default | Effect |
|
|
190
|
-
| --- | --- | --- |
|
|
191
|
-
| `--job NAME` | every job | Run only this job, ignoring its `refresh` window. Its predecessors don't run; `run` warns about each. Repeatable. |
|
|
192
|
-
| `--force` | off | Ignore every job's `refresh` window. |
|
|
193
|
-
| `--forever` | off | Keep running cycles instead of exiting after one. For freshness under cron's one-minute floor. |
|
|
194
|
-
| `--dry-run` | off | Check connections, target tables, primary keys and masking coverage, moving no rows. |
|
|
195
|
-
| `--accept-key-change` | off | Run upsert jobs whose masking key changed since their last run. Refused where the policy masks the target's primary key; see [rotating the masking key](docs/operations.md#rotating-the-masking-key). |
|
|
196
|
-
| `--notify-url URL` | `$BAUTA_NOTIFY_URL` | Post a JSON summary to this webhook when a cycle doesn't succeed. |
|
|
197
|
-
|
|
198
|
-
### Run state, history and the manifest
|
|
199
|
-
|
|
200
|
-
Each is kept in a file or in a database table, set in [`jobs.yaml`](docs/configuration.md#file-level) or overridden for one run by its flags. A file set in `jobs.yaml` is relative to `jobs.yaml`; a file flag is relative to the working directory.
|
|
201
|
-
|
|
202
|
-
| What | `jobs.yaml` setting | File | Table | Without either |
|
|
203
|
-
| --- | --- | --- | --- | --- |
|
|
204
|
-
| **Run state**: last runs, watermarks and key fingerprints | `memory` | `--memory FILE` | `--memory-connection ALIAS`, `--memory-table NAME` | `memory.yaml` beside `jobs.yaml` |
|
|
205
|
-
| **History**: one record per job per cycle, for `bauta history` | `history` | `--history FILE` | `--history-connection ALIAS`, `--history-table NAME` | Not recorded. |
|
|
206
|
-
| **Masking manifest**: what was masked and how, sealed, for `bauta verify-manifest` | `manifest` | `--manifest FILE` | `--manifest-connection ALIAS`, `--manifest-table NAME` | Not written. |
|
|
207
|
-
|
|
208
|
-
Tables default to `bauta_memory`, `bauta_history` and `bauta_manifest`, and must exist first; [operations.md](docs/operations.md#tables) has their definitions. A manifest is signed when `$BAUTA_MANIFEST_KEY` is set.
|
|
209
|
-
|
|
210
|
-
### Saying less, and saying it once
|
|
211
|
-
|
|
212
|
-
A `defaults:` block in [`jobs.yaml`](docs/configuration.md#defaults) supplies what every job would otherwise repeat, so a job says only what is particular to it:
|
|
213
|
-
|
|
214
|
-
```yaml
|
|
215
|
-
defaults:
|
|
216
|
-
sourceConnection: prod
|
|
217
|
-
targetConnection: staging
|
|
218
|
-
insertStrategy: upsert
|
|
219
|
-
masking:
|
|
220
|
-
key: ${MASKING_KEY}
|
|
221
179
|
|
|
222
|
-
|
|
223
|
-
maskCustomers:
|
|
224
|
-
sourceQuery: select id, email from customers
|
|
225
|
-
targetTableFinal: customers
|
|
226
|
-
masking:
|
|
227
|
-
columns: {id: keep, email: email}
|
|
228
|
-
```
|
|
229
|
-
|
|
230
|
-
A masking policy's `columns` stays with its job, so what a job does to its data can be read in one place. Any field a file doesn't recognise is an error, so a misspelling stops `bauta validate` rather than being quietly ignored.
|
|
231
|
-
|
|
232
|
-
### Proving the copy is safe
|
|
233
|
-
|
|
234
|
-
| What | Where |
|
|
235
|
-
| --- | --- |
|
|
236
|
-
| Every column of every job is covered, and references still match once masked | `bauta audit --connect --strict` |
|
|
237
|
-
| Every table in production is copied, declared, or fails the build | `bauta coverage` |
|
|
238
|
-
| Nothing can ever reach this database unmasked | [`requireMasking`](docs/configuration.md#requiring-masking) on the alias |
|
|
239
|
-
| This job copies as it stands, and somebody decided so | [`unmasked: true`](docs/configuration.md#copying-without-masking) on the job |
|
|
240
|
-
| This table is deliberately not copied, and why | [`acknowledged`](docs/configuration.md#acknowledged) in `jobs.yaml` |
|
|
241
|
-
| What was masked, how, and under which key, sealed | `bauta verify-manifest` |
|
|
242
|
-
| No row in the copy points at a row that isn't there | `bauta verify-references` |
|
|
243
|
-
|
|
244
|
-
### Proposing and reviewing policies
|
|
245
|
-
|
|
246
|
-
`discover`, `subset --mask`, `audit` and `synthesize` recognise personal data by built-in rules, and by rules of your own:
|
|
247
|
-
|
|
248
|
-
| Flag | Default | Effect |
|
|
249
|
-
| --- | --- | --- |
|
|
250
|
-
| `--rules FILE` | `discovery.yaml` in the configuration directory, if there is one | Check these rules before the built-in ones. See [your own rules](docs/masking.md#your-own-rules-discoveryyaml). |
|
|
251
|
-
|
|
252
|
-
`discover` takes `--all-tables` for a whole database instead of a repeated `--table`, and `discover`/`subset` take `--mask-keys` to mask numeric surrogate keys as well as text ones, in the domain each foreign key already shares.
|
|
253
|
-
|
|
254
|
-
### Environment variables
|
|
255
|
-
|
|
256
|
-
The ones you'd set in a deployment; [operations.md](docs/operations.md#environment-variables) also lists two for diagnosis.
|
|
257
|
-
|
|
258
|
-
| Variable | Effect |
|
|
259
|
-
| --- | --- |
|
|
260
|
-
| `BAUTA_CONFIG` | The configuration directory, when `--config` isn't given. |
|
|
261
|
-
| `BAUTA_NOTIFY_URL` | The webhook, when `--notify-url` isn't given. |
|
|
262
|
-
| `BAUTA_MANIFEST_KEY` | Sign manifests, and verify their signatures. |
|
|
263
|
-
| `BAUTA_MASKING_THREADS` | Threads the native masker uses per job: a number or `auto`. Overrides `jobs.yaml`'s `maskingThreads`; see [masking threads](docs/masking.md#masking-threads). |
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
## Documentation
|
|
267
|
-
|
|
268
|
-
| Document | Covers |
|
|
269
|
-
| --- | --- |
|
|
270
|
-
| [Configuration](docs/configuration.md) | every field, how credentials are read from the environment, and connection options such as TLS |
|
|
271
|
-
| [Masking](docs/masking.md) | strategies, consistent masks across tables, the key, the manifest, `audit`, `discover`, `subset`, `schema`, `synthesize` and `clear` |
|
|
272
|
-
| [How it works](docs/design.md) | streaming, incremental loads, retries, scheduling, and the masking design |
|
|
273
|
-
| [Operating it](docs/operations.md) | run state, history and notifications |
|
|
274
|
-
| [Security model](docs/security.md) | what masking protects and what it doesn't, the constructions, keys, and a deployment checklist |
|
|
275
|
-
| [Library](docs/library.md) | embedding it in Python, results, memory backends |
|
|
276
|
-
| [Development](docs/development.md) | running the tests, including against real databases; see also [contributing](CONTRIBUTING.md) |
|
|
277
|
-
| [Changelog](CHANGELOG.md) | what changed in each release, breaking changes first |
|
|
180
|
+
Every command and flag is on its own page under [commands](docs/reference/commands/index.md).
|
|
278
181
|
|
|
279
182
|
|
|
280
183
|
## Layout
|
|
@@ -293,7 +196,7 @@ The ones you'd set in a deployment; [operations.md](docs/operations.md#environme
|
|
|
293
196
|
| `mask-rs/` | the optional native masker, in Rust — see [its README](mask-rs/README.md) |
|
|
294
197
|
| `example/` | runnable demos, each with its `configuration/`, and a starter configuration — see [its README](example/README.md) |
|
|
295
198
|
| `docs/` | the documentation above |
|
|
296
|
-
| `tests/` | the test suite, laid out like the package; see [where the tests are](docs/development.md#where-the-tests-are) |
|
|
199
|
+
| `tests/` | the test suite, laid out like the package; see [where the tests are](docs/project/development.md#where-the-tests-are) |
|
|
297
200
|
|
|
298
201
|
|
|
299
202
|
## License
|
bauta-0.2.4/README.md
ADDED
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
# Bauta
|
|
2
|
+
|
|
3
|
+
**Put a mask on production:** a safe, realistic stand-in for your production data, in whichever database you need it. Bauta masks what it copies, copies only the slice you need with every relationship intact, generates what may not be copied at all, and moves data between databases on a schedule you already run, with nothing to host.
|
|
4
|
+
|
|
5
|
+
- **Masking:** consistent across tables and runs, one-to-one for keys (NIST FF1 where policy requires it), applied before anything reaches the target, and every column must be covered.
|
|
6
|
+
- **Discovery, subsets and synthetic data:** propose a masking policy from a live schema, copy a referentially complete slice of production, create the copy's tables in whichever database it goes to, and fill tables that can't be copied with generated rows.
|
|
7
|
+
- **Audit:** report what every job does with data and what a reviewer should question, and seal each run's masking manifest so it can be verified later.
|
|
8
|
+
- **Provable coverage:** list every table in production and what the jobs do with each, so a table nobody wrote a job for fails the build rather than going unnoticed. A database can require that nothing reaches it unmasked.
|
|
9
|
+
- **Seven databases:** Oracle, SQL Server, PostgreSQL, MySQL, MariaDB, SQLite and DuckDB, as source or target in any combination.
|
|
10
|
+
- **Files as a target:** a masked copy in Parquet, CSV or JSON Lines, in a directory or in S3, Google Cloud Storage or Azure, for a data lake that Athena, Snowflake or Databricks reads: appended to incrementally or published whole as a snapshot, with nothing visible until a run succeeds.
|
|
11
|
+
- **Iceberg tables:** a masked copy as Iceberg tables in any of the three clouds, through a Glue, REST or SQL catalog: each run one commit, upserts that merge by key, and old snapshots' files deleted with them.
|
|
12
|
+
- **Streaming:** memory stays flat however large the table, and PostgreSQL and SQL Server targets load in bulk.
|
|
13
|
+
- **Fast:** a million rows of six masked columns, two of them one-to-one keys, in under 8 seconds on one core with the optional native masker, and 74 without; more cores for wide tables. Either way the masks are the same.
|
|
14
|
+
- **Incremental loads:** extract only what changed since the last successful run.
|
|
15
|
+
- **A dependency graph:** jobs run in order, concurrently where they can, each in its own process with an optional timeout.
|
|
16
|
+
- **Operable:** webhook alerts; run state, history and manifests each in a file or a table; and passwords from a command for cloud IAM tokens.
|
|
17
|
+
- **No infrastructure:** a `pip install`, some YAML, and a command you run from cron.
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
## Documentation
|
|
21
|
+
|
|
22
|
+
**[ribeiro11075.github.io/bauta](https://ribeiro11075.github.io/bauta/)**, for the release you have installed, or read it here in [`docs/`](docs):
|
|
23
|
+
|
|
24
|
+
| Start with | For |
|
|
25
|
+
| --- | --- |
|
|
26
|
+
| [Install](docs/get-started/install.md) and [Quickstart](docs/get-started/quickstart.md) | the drivers, the native masker, and a first run |
|
|
27
|
+
| [Guides](docs/guides/mask-a-table.md) | one task each: masking a table, keeping joins, subsets, proving the copy safe, running it on a schedule |
|
|
28
|
+
| [Reference](docs/reference/configuration.md) | every file, field, strategy and [command](docs/reference/commands/index.md) |
|
|
29
|
+
| [How it works](docs/concepts/how-it-works.md) and the [security model](docs/concepts/security.md) | the design, and what masking protects and what it doesn't |
|
|
30
|
+
| [Changelog](CHANGELOG.md) | what changed in each release, breaking changes first |
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
## Install
|
|
34
|
+
|
|
35
|
+
Python 3.10 or newer. Choose the drivers you need as extras; each is loaded only when a connection uses it.
|
|
36
|
+
|
|
37
|
+
```
|
|
38
|
+
pip install "bauta[postgresql,oracle]"
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
[Install](docs/get-started/install.md) lists every extra, and the optional native masker, which masks about ten times as fast.
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
## Quickstart
|
|
45
|
+
|
|
46
|
+
Start from the configuration in [`example/starter/configuration/`](example/starter/configuration), from a clone or downloaded from GitHub:
|
|
47
|
+
|
|
48
|
+
```
|
|
49
|
+
mkdir configuration
|
|
50
|
+
cp example/starter/configuration/*.yaml configuration/
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
Edit `configuration/connections.yaml` and `configuration/jobs.yaml` for your databases, then supply the credentials they reference:
|
|
54
|
+
|
|
55
|
+
```
|
|
56
|
+
export SOURCE_DB_PASSWORD=... TARGET_DB_PASSWORD=... MASKING_KEY=...
|
|
57
|
+
|
|
58
|
+
bauta validate # check the configuration, offline
|
|
59
|
+
bauta run --dry-run # check connections and tables, moving nothing
|
|
60
|
+
bauta run # run every job once
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
To see it work without any of that, the demos in a clone of this repository use throwaway SQLite databases:
|
|
64
|
+
|
|
65
|
+
```
|
|
66
|
+
git clone https://github.com/ribeiro11075/bauta.git && cd bauta
|
|
67
|
+
pip install -e ".[fpe]"
|
|
68
|
+
|
|
69
|
+
python example/walkthrough/demo.py # the whole workflow: discover, subset, audit, mask, verify, synthesize
|
|
70
|
+
python example/incremental/demo.py # streaming and incremental loads
|
|
71
|
+
python example/masking/demo.py # masking, discovery and a subset, from Python
|
|
72
|
+
python example/native-masking/demo.py # Python against Rust, and Rust on one core against all of them
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
## The command
|
|
77
|
+
|
|
78
|
+
```
|
|
79
|
+
bauta run run every job that's due, once
|
|
80
|
+
bauta validate check the configuration, without connecting
|
|
81
|
+
bauta jobs show the job graph and which jobs are due
|
|
82
|
+
bauta history show recent job outcomes
|
|
83
|
+
|
|
84
|
+
bauta discover propose a masking policy for tables
|
|
85
|
+
bauta subset generate jobs that copy a referentially complete subset
|
|
86
|
+
bauta schema create target tables from source ones, in the target's dialect
|
|
87
|
+
bauta synthesize fill tables with generated rows, for data that can't be copied
|
|
88
|
+
bauta clear empty the jobs' target tables, children first
|
|
89
|
+
|
|
90
|
+
bauta audit report what each job does with data, and what to question
|
|
91
|
+
bauta coverage list a source database's tables and what the jobs do with each
|
|
92
|
+
bauta verify-manifest check a masking manifest is unaltered, and who signed it
|
|
93
|
+
bauta verify-references count rows in the copy whose foreign key points at nothing
|
|
94
|
+
|
|
95
|
+
bauta --version print the version, and which masker it would use
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
| Exit code | Meaning |
|
|
99
|
+
| --- | --- |
|
|
100
|
+
| `0` | Every job completed. |
|
|
101
|
+
| `1` | A job failed, or was skipped because a predecessor failed, or the command failed on a database error. |
|
|
102
|
+
| `2` | Invalid configuration or usage. |
|
|
103
|
+
| `130` | Interrupted by a signal: running jobs finished, the rest were skipped. |
|
|
104
|
+
|
|
105
|
+
`run` makes one pass and exits, so it fits under cron or a Kubernetes CronJob. A second `run` sharing the same run state refuses to start while the first is still going. `bauta <command> --help` lists every flag.
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
Every command and flag is on its own page under [commands](docs/reference/commands/index.md).
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
## Layout
|
|
112
|
+
|
|
113
|
+
| Path | What it is |
|
|
114
|
+
| --- | --- |
|
|
115
|
+
| `bauta/configuration/` | reading and validating the YAML: `${NAME}` and `passwordCommand` in `environment.py`, the models in `models.py` |
|
|
116
|
+
| `bauta/database/` | streaming and loading rows in `connection.py`; what differs between the seven databases in `dialects/`, one module each |
|
|
117
|
+
| `bauta/jobs/` | `runner.py` runs a cycle of jobs, each in a process of its own (`workers.py`) moving rows a chunk at a time (`pipeline.py`); run state in `memory.py`, history and manifests in `reporting.py` |
|
|
118
|
+
| `bauta/masking/` | the keyed hash, `Strategy` and masking plans in `core.py`, the built-in strategies in `strategies.py` |
|
|
119
|
+
| `bauta/transform/` | per-column transforms, applied before masking, and the ones that ship in `builtinTransforms.py` |
|
|
120
|
+
| `bauta/generate/` | `discover`, `subset`, `schema` and `synthesize`: what is built from a live schema |
|
|
121
|
+
| `bauta/review/` | `audit`, `coverage` and `verify-references`: reports that move no data |
|
|
122
|
+
| `bauta/log/` | logging, and the scrubbing that keeps values out of every message |
|
|
123
|
+
| `bauta/cli/` | the `bauta` command, one module per group of subcommands |
|
|
124
|
+
| `mask-rs/` | the optional native masker, in Rust — see [its README](mask-rs/README.md) |
|
|
125
|
+
| `example/` | runnable demos, each with its `configuration/`, and a starter configuration — see [its README](example/README.md) |
|
|
126
|
+
| `docs/` | the documentation above |
|
|
127
|
+
| `tests/` | the test suite, laid out like the package; see [where the tests are](docs/project/development.md#where-the-tests-are) |
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
## License
|
|
131
|
+
|
|
132
|
+
[MIT](LICENSE)
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
"""Safe, realistic copies of production across databases. Most deployments
|
|
2
|
-
use the `bauta` command; see docs/
|
|
2
|
+
use the `bauta` command; see docs/reference/python-api.md for embedding it.
|
|
3
3
|
|
|
4
|
-
What is importable from here is the public API, the one docs/
|
|
4
|
+
What is importable from here is the public API, the one docs/reference/python-api.md
|
|
5
5
|
describes. A name reached only through a submodule is internal, unless the
|
|
6
6
|
documentation names it there, and may change in any release.
|
|
7
7
|
"""
|
|
8
8
|
from .configuration import (Configuration, ConfigurationError, ConnectionConfig, DatabaseType, DataJobConfig, DataJobsFile, DiscoveryRulesFile,
|
|
9
9
|
DuckDBConnection, FilesConnection, InsertStrategy, MariaDBConnection, MaskingConfig, MSSQLConnection, MySQLConnection, OracleConnection,
|
|
10
|
-
PostgreSQLConnection, SQLiteConnection, connectionConfig, expandEnvironmentVariables)
|
|
10
|
+
PostgreSQLConnection, SQLiteConnection, connectionConfig, expandEnvironmentVariables, readJobsFile)
|
|
11
11
|
from .database import Database
|
|
12
12
|
from .jobs.dependencyGraph import DependencyGraph, JobOutcome, JobStatus
|
|
13
13
|
from .database.dialects import ForeignKey
|
|
@@ -27,6 +27,7 @@ __all__ = [
|
|
|
27
27
|
'auditJobs',
|
|
28
28
|
'buildMaskingManifest',
|
|
29
29
|
'Configuration',
|
|
30
|
+
'readJobsFile',
|
|
30
31
|
'ConfigurationError',
|
|
31
32
|
'Database',
|
|
32
33
|
'DATABASE_HISTORY_SCHEMA',
|
|
@@ -47,7 +47,7 @@ def _addCommonArguments(parser: argparse.ArgumentParser, jobs: bool = True, memo
|
|
|
47
47
|
if jobs and memory:
|
|
48
48
|
parser.add_argument('--memory', help='path to the run-memory file (default: jobs.yaml\'s `memory`, else memory.yaml beside jobs.yaml)')
|
|
49
49
|
parser.add_argument('--memory-connection', metavar='ALIAS',
|
|
50
|
-
help='keep run memory in a table of this connection instead of a file (see docs/
|
|
50
|
+
help='keep run memory in a table of this connection instead of a file (see docs/guides/run-on-a-schedule.md)')
|
|
51
51
|
parser.add_argument('--memory-table', help='the run-memory table (default: jobs.yaml\'s, else bauta_memory)')
|
|
52
52
|
_addLoggingArguments(parser)
|
|
53
53
|
|
|
@@ -73,11 +73,11 @@ def _addManifestKeyArgument(parser: argparse.ArgumentParser) -> None:
|
|
|
73
73
|
|
|
74
74
|
def _addLoggingArguments(parser: argparse.ArgumentParser) -> None:
|
|
75
75
|
|
|
76
|
-
parser.add_argument('--log', help='also write logs to this file (logs always go to stderr
|
|
76
|
+
parser.add_argument('--log', help='also write logs to this file (logs always go to stderr; --quiet leaves only errors there)')
|
|
77
77
|
parser.add_argument('--log-level', default='info', choices=['debug', 'info', 'warning', 'error'], help='default: info')
|
|
78
78
|
parser.add_argument('--log-format', default='text', choices=['text', 'json'],
|
|
79
79
|
help='json emits one object per record, carrying job/status/rowCount as fields a log collector can filter and alert on')
|
|
80
|
-
parser.add_argument('--quiet', action='store_true', help='
|
|
80
|
+
parser.add_argument('--quiet', action='store_true', help='log only errors to stderr, one line each, without tracebacks')
|
|
81
81
|
|
|
82
82
|
|
|
83
83
|
def _positiveInteger(text: str) -> int:
|
|
@@ -89,6 +89,15 @@ def _positiveInteger(text: str) -> int:
|
|
|
89
89
|
return value
|
|
90
90
|
|
|
91
91
|
|
|
92
|
+
def _nonNegativeInteger(text: str) -> int:
|
|
93
|
+
|
|
94
|
+
value = int(text)
|
|
95
|
+
if value < 0:
|
|
96
|
+
raise argparse.ArgumentTypeError('must be at least 0, got {}'.format(value))
|
|
97
|
+
|
|
98
|
+
return value
|
|
99
|
+
|
|
100
|
+
|
|
92
101
|
def _addRunArguments(parser: argparse.ArgumentParser) -> None:
|
|
93
102
|
|
|
94
103
|
parser.add_argument('--forever', action='store_true',
|
|
@@ -112,6 +121,9 @@ def _addGeneratorArguments(parser: argparse.ArgumentParser) -> None:
|
|
|
112
121
|
parser.add_argument('--sample', type=_positiveInteger, default=1000, help='rows sampled per table to classify columns (default: 1000)')
|
|
113
122
|
parser.add_argument('--key-variable', default='MASKING_KEY', help='environment variable the generated jobs read the masking key from')
|
|
114
123
|
parser.add_argument('--chunk-size', type=_positiveInteger, default=5000, help='chunkSize for the generated jobs (default: 5000)')
|
|
124
|
+
parser.add_argument('--self-contained', action='store_true',
|
|
125
|
+
help='write the connections, insert strategy and key into every job rather than under defaults:, '
|
|
126
|
+
'for jobs to paste into a file with defaults of its own')
|
|
115
127
|
parser.add_argument('--mask-keys', action='store_true',
|
|
116
128
|
help='mask numeric surrogate keys too, in the domain each foreign key shares, instead of keeping them')
|
|
117
129
|
parser.add_argument('--output', help='write the generated jobs here instead of stdout; must not already exist')
|
|
@@ -154,6 +166,9 @@ def _buildParser() -> argparse.ArgumentParser:
|
|
|
154
166
|
help='write a JSON record of what was masked, how, and under which key fingerprint (default: jobs.yaml\'s `manifest`)')
|
|
155
167
|
_addManifestLocationArguments(runParser)
|
|
156
168
|
_addManifestKeyArgument(runParser)
|
|
169
|
+
runParser.add_argument('--full-refresh', action='store_true',
|
|
170
|
+
help='load every incremental job\'s whole source and replace its target, so rows deleted from the source '
|
|
171
|
+
'leave the copy; implies --force. Database targets need targetTableStage')
|
|
157
172
|
runParser.add_argument('--accept-key-change', action='store_true',
|
|
158
173
|
help='run upsert jobs even though their masking key changed since their last run')
|
|
159
174
|
_addHistoryArguments(runParser)
|
|
@@ -173,7 +188,10 @@ def _buildParser() -> argparse.ArgumentParser:
|
|
|
173
188
|
auditParser.add_argument('--connect', action='store_true',
|
|
174
189
|
help='also run each masked query for its real columns, and check whether each connection is encrypted')
|
|
175
190
|
auditParser.add_argument('--job', action='append', help='audit only this job (repeatable)')
|
|
176
|
-
auditParser.add_argument('--
|
|
191
|
+
auditParser.add_argument('--sample', type=_nonNegativeInteger, default=1000, metavar='ROWS',
|
|
192
|
+
help='with --connect, rows of each query read to check kept columns and shared domains by their values; 0 reads none')
|
|
193
|
+
auditParser.add_argument('--format', default='text', choices=['text', 'json', 'html'],
|
|
194
|
+
help='text, json, or html: one self-contained page for a reviewer, best with --output (default: text)')
|
|
177
195
|
auditParser.add_argument('--strict', action='store_true', help='exit 1 on warnings as well as errors')
|
|
178
196
|
auditParser.add_argument('--output', help='write the report here instead of stdout; must not already exist')
|
|
179
197
|
_addRulesArgument(auditParser)
|
|
@@ -192,7 +210,8 @@ def _buildParser() -> argparse.ArgumentParser:
|
|
|
192
210
|
coverageParser.add_argument('--connection', help='the source connection alias to check; required when the jobs read from more than one')
|
|
193
211
|
coverageParser.add_argument('--schema', help='the schema to list, instead of the connection\'s own')
|
|
194
212
|
coverageParser.add_argument('--job', action='append', help='only these jobs count as covering a table. Repeatable.')
|
|
195
|
-
coverageParser.add_argument('--format', choices=['text', 'json'], default='text',
|
|
213
|
+
coverageParser.add_argument('--format', choices=['text', 'json', 'html'], default='text',
|
|
214
|
+
help='text, json, or html: one self-contained page for a reviewer, best with --output (default: text)')
|
|
196
215
|
coverageParser.add_argument('--output', help='write to this file instead of stdout')
|
|
197
216
|
|
|
198
217
|
verifyParser = subparsers.add_parser('verify-manifest', help='check that a manifest is unaltered, and who signed it')
|
|
@@ -216,14 +235,21 @@ def _buildParser() -> argparse.ArgumentParser:
|
|
|
216
235
|
jobsParser.set_defaults(handler=_commandJobs)
|
|
217
236
|
|
|
218
237
|
discoverParser = subparsers.add_parser('discover', help='propose a masking policy for tables, from their schema and a sample')
|
|
219
|
-
_addCommonArguments(discoverParser,
|
|
220
|
-
discoverParser.add_argument('--connection',
|
|
238
|
+
_addCommonArguments(discoverParser, memory=False)
|
|
239
|
+
discoverParser.add_argument('--connection', help='the alias to read from; required unless --update')
|
|
221
240
|
discoverParser.add_argument('--table', action='append', help='a table to propose a policy for (repeatable)')
|
|
222
241
|
discoverParser.add_argument('--all-tables', action='store_true', help='every table in the database, instead of naming each with --table')
|
|
223
242
|
discoverParser.add_argument('--schema', help='the schema --all-tables lists, instead of the connection\'s own')
|
|
224
|
-
discoverParser.add_argument('--target', help='the alias the generated jobs load into (default: --
|
|
243
|
+
discoverParser.add_argument('--target', help='the alias the generated jobs load into (default: --connection, masking in place)')
|
|
225
244
|
_addGeneratorArguments(discoverParser)
|
|
226
245
|
_addRulesArgument(discoverParser)
|
|
246
|
+
discoverParser.add_argument('--update', action='store_true',
|
|
247
|
+
help='compare each masked job in the jobs file with what its sourceQuery returns now, and propose a policy '
|
|
248
|
+
'for each new column; exits 1 if any job has drifted')
|
|
249
|
+
discoverParser.add_argument('--job', action='append', help='with --update, only this job (repeatable)')
|
|
250
|
+
discoverParser.add_argument('--apply', action='store_true',
|
|
251
|
+
help='with --update, write the proposals into the file each job is defined in, and drop the columns '
|
|
252
|
+
'its sourceQuery no longer returns')
|
|
227
253
|
discoverParser.set_defaults(handler=_commandDiscover)
|
|
228
254
|
|
|
229
255
|
subsetParser = subparsers.add_parser('subset', help='generate jobs that copy a referentially complete subset')
|
|
@@ -244,11 +270,13 @@ def _buildParser() -> argparse.ArgumentParser:
|
|
|
244
270
|
_addCommonArguments(schemaParser, jobs=False)
|
|
245
271
|
schemaParser.add_argument('--connection', required=True, help='the alias to read table definitions from')
|
|
246
272
|
schemaParser.add_argument('--target', required=True, help='the alias the tables are for; its dialect decides the types')
|
|
247
|
-
schemaParser.add_argument('--table', action='append',
|
|
273
|
+
schemaParser.add_argument('--table', action='append', help='a table to create (repeatable)')
|
|
274
|
+
schemaParser.add_argument('--all-tables', action='store_true', help='every table in the database, instead of naming each with --table')
|
|
275
|
+
schemaParser.add_argument('--schema', help='the schema --all-tables lists, instead of the connection\'s own')
|
|
248
276
|
schemaParser.add_argument('--related', action='store_true', help='also every table a subset rooted at --table would copy')
|
|
249
277
|
schemaParser.add_argument('--no-children', action='store_true', help='with --related, only the tables --table references')
|
|
250
278
|
schemaParser.add_argument('--no-foreign-keys', action='store_true', help='leave foreign keys out of the generated tables')
|
|
251
|
-
schemaParser.add_argument('--stage-suffix', help='also create <table><suffix> stage tables, for swap jobs; alone if --target is --
|
|
279
|
+
schemaParser.add_argument('--stage-suffix', help='also create <table><suffix> stage tables, for swap jobs; alone if --target is --connection')
|
|
252
280
|
schemaParser.add_argument('--apply', action='store_true', help='create the tables in --target, skipping any that already exist')
|
|
253
281
|
schemaParser.add_argument('--output', help='write the SQL here instead of stdout; must not already exist')
|
|
254
282
|
schemaParser.set_defaults(handler=_commandSchema)
|
|
@@ -14,7 +14,7 @@ from typing import Any, Dict, Generator, List, Mapping, Optional, Sequence, Tupl
|
|
|
14
14
|
import yaml
|
|
15
15
|
|
|
16
16
|
from ..configuration import (Configuration, ConfigurationError, ConnectionConfig, DatabaseConfig, DataJobConfig, DataJobsFile, FilesConnection, IcebergConnection, isLake,
|
|
17
|
-
StorageLocation, TableLocation, expandEnvironmentVariables)
|
|
17
|
+
JobsDocument, StorageLocation, TableLocation, expandEnvironmentVariables, readJobsFile)
|
|
18
18
|
from ..database import Database
|
|
19
19
|
from ..log import Log
|
|
20
20
|
from ..masking import MaskingError
|
|
@@ -58,6 +58,12 @@ def _loadYaml(path: Path) -> Any:
|
|
|
58
58
|
raise UsageError('{} is not valid YAML: {}'.format(path, error)) from error
|
|
59
59
|
|
|
60
60
|
|
|
61
|
+
def _readJobs(jobsPath: Path) -> JobsDocument:
|
|
62
|
+
"""The jobs file with the files its `include` names merged in."""
|
|
63
|
+
|
|
64
|
+
return readJobsFile(jobsPath, load=_loadYaml)
|
|
65
|
+
|
|
66
|
+
|
|
61
67
|
CONNECTIONS_FILE = 'connections.yaml'
|
|
62
68
|
|
|
63
69
|
|
|
@@ -162,12 +168,16 @@ def _history(location: Location, connectionConfiguration: Dict[str, ConnectionCo
|
|
|
162
168
|
|
|
163
169
|
|
|
164
170
|
def _configureLogging(arguments: argparse.Namespace) -> Log:
|
|
165
|
-
"""stderr
|
|
171
|
+
"""stderr, where a container collects it; --log adds a file. --quiet
|
|
172
|
+
leaves only errors on stderr, one line each.
|
|
173
|
+
"""
|
|
166
174
|
|
|
167
175
|
level = getattr(logging, arguments.log_level.upper())
|
|
168
176
|
log = Log(logFile=arguments.log, level=level, logFormat=arguments.log_format)
|
|
169
177
|
|
|
170
|
-
if
|
|
178
|
+
if arguments.quiet:
|
|
179
|
+
log.addErrorStream(stream=sys.stderr)
|
|
180
|
+
else:
|
|
171
181
|
log.addStreamHandler(stream=sys.stderr, level=level)
|
|
172
182
|
|
|
173
183
|
return log
|
|
@@ -177,14 +187,14 @@ def _loadConnections(arguments: argparse.Namespace) -> Dict[str, ConnectionConfi
|
|
|
177
187
|
|
|
178
188
|
_, connectionsPath = _resolveConfigurationPaths(arguments)
|
|
179
189
|
|
|
180
|
-
return Configuration.validateConnectionConfiguration(_loadYaml(connectionsPath))
|
|
190
|
+
return Configuration.validateConnectionConfiguration(_loadYaml(connectionsPath), directory=str(connectionsPath.parent))
|
|
181
191
|
|
|
182
192
|
|
|
183
193
|
def _loadDataJobs(arguments: argparse.Namespace) -> Tuple[DataJobsFile, Dict[str, ConnectionConfig]]:
|
|
184
194
|
|
|
185
195
|
jobsPath, connectionsPath = _resolveConfigurationPaths(arguments)
|
|
186
|
-
connectionConfiguration = Configuration.validateConnectionConfiguration(_loadYaml(connectionsPath))
|
|
187
|
-
jobsFile = Configuration.validateJobConfiguration(
|
|
196
|
+
connectionConfiguration = Configuration.validateConnectionConfiguration(_loadYaml(connectionsPath), directory=str(connectionsPath.parent))
|
|
197
|
+
jobsFile = Configuration.validateJobConfiguration(_readJobs(jobsPath).content, DataJobsFile)
|
|
188
198
|
Configuration.validateJobGraph(jobsFile.jobs, connections=connectionConfiguration)
|
|
189
199
|
|
|
190
200
|
unknown = ['{}: connection "{}" is not a known connection alias'.format(setting, location.connection)
|
|
@@ -290,6 +300,25 @@ def _sourceQueryColumns(job: DataJobConfig, connections: _Connections) -> List[s
|
|
|
290
300
|
return columns
|
|
291
301
|
|
|
292
302
|
|
|
303
|
+
def _sourceQuerySample(job: DataJobConfig, connections: _Connections, rows: int) -> Tuple[List[str], List[Tuple[Any, ...]]]:
|
|
304
|
+
"""The columns a job's sourceQuery returns, and up to `rows` of its rows
|
|
305
|
+
from the start, without reading the rest. They stay in memory: nothing
|
|
306
|
+
that reads them may report a value.
|
|
307
|
+
"""
|
|
308
|
+
|
|
309
|
+
with connections.use(job.sourceConnection) as database:
|
|
310
|
+
query = job.sourceQuery
|
|
311
|
+
parameters = None
|
|
312
|
+
if job.watermarkColumn:
|
|
313
|
+
query = database.substituteWatermarkPlaceholder(query)
|
|
314
|
+
parameters = (job.watermarkInitial,)
|
|
315
|
+
columns, chunks = database.stream(query=query, chunkSize=max(1, rows), parameters=parameters)
|
|
316
|
+
with chunks:
|
|
317
|
+
sampled = next(chunks, []) if rows else []
|
|
318
|
+
|
|
319
|
+
return columns, list(sampled)
|
|
320
|
+
|
|
321
|
+
|
|
293
322
|
def _targetColumns(job: DataJobConfig, connections: _Connections) -> List[str]:
|
|
294
323
|
"""The target's columns, in the order a load fills them."""
|
|
295
324
|
|