bauta 0.2.0__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {bauta-0.2.0 → bauta-0.2.2}/PKG-INFO +12 -7
- {bauta-0.2.0 → bauta-0.2.2}/README.md +3 -1
- {bauta-0.2.0 → bauta-0.2.2}/bauta/cli/common.py +8 -7
- {bauta-0.2.0 → bauta-0.2.2}/bauta/cli/generate.py +5 -3
- {bauta-0.2.0 → bauta-0.2.2}/bauta/cli/review.py +7 -6
- {bauta-0.2.0 → bauta-0.2.2}/bauta/cli/run.py +49 -51
- {bauta-0.2.0 → bauta-0.2.2}/bauta/configuration/__init__.py +17 -8
- bauta-0.2.2/bauta/configuration/connections.py +786 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/configuration/models.py +132 -631
- {bauta-0.2.0 → bauta-0.2.2}/bauta/database/connection.py +23 -9
- {bauta-0.2.0 → bauta-0.2.2}/bauta/database/values.py +131 -28
- {bauta-0.2.0 → bauta-0.2.2}/bauta/jobs/pipeline.py +25 -21
- {bauta-0.2.0 → bauta-0.2.2}/bauta/jobs/runner.py +3 -2
- {bauta-0.2.0 → bauta-0.2.2}/bauta/jobs/targets.py +4 -1
- bauta-0.2.2/bauta/lake/__init__.py +30 -0
- bauta-0.2.2/bauta/lake/columnar.py +204 -0
- {bauta-0.2.0/bauta/files → bauta-0.2.2/bauta/lake}/columns.py +21 -8
- {bauta-0.2.0/bauta/files → bauta-0.2.2/bauta/lake}/formats.py +135 -15
- bauta-0.2.2/bauta/lake/iceberg.py +499 -0
- {bauta-0.2.0/bauta/files → bauta-0.2.2/bauta/lake}/stores.py +15 -55
- bauta-0.2.2/bauta/lake/target.py +204 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/masking/core.py +79 -11
- {bauta-0.2.0 → bauta-0.2.2}/bauta.egg-info/PKG-INFO +12 -7
- {bauta-0.2.0 → bauta-0.2.2}/bauta.egg-info/SOURCES.txt +9 -6
- {bauta-0.2.0 → bauta-0.2.2}/bauta.egg-info/requires.txt +9 -5
- {bauta-0.2.0 → bauta-0.2.2}/pyproject.toml +11 -6
- bauta-0.2.0/bauta/files/__init__.py +0 -11
- bauta-0.2.0/bauta/files/target.py +0 -419
- {bauta-0.2.0 → bauta-0.2.2}/LICENSE +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/__init__.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/cli/__init__.py +0 -0
- /bauta-0.2.0/bauta/configuration/fileTypes.py → /bauta-0.2.2/bauta/configuration/columnTypes.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/configuration/environment.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/database/__init__.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/database/dialects/__init__.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/database/dialects/base.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/database/dialects/duckdb.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/database/dialects/mssql.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/database/dialects/mysql.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/database/dialects/names.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/database/dialects/oracle.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/database/dialects/postgresql.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/database/dialects/sqlite.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/database/driver.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/generate/__init__.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/generate/builtinDiscovery.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/generate/discovery.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/generate/schema.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/generate/subset.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/generate/synthesize.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/jobs/__init__.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/jobs/dependencyGraph.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/jobs/keys.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/jobs/memory.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/jobs/reporting.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/jobs/workers.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/log/__init__.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/log/core.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/log/scrubbing.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/masking/__init__.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/masking/fakeData.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/masking/fpe.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/masking/strategies.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/py.typed +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/review/__init__.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/review/audit.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/review/coverage.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/review/references.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/transform/__init__.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/transform/builtinTransforms.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta/transform/core.py +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta.egg-info/dependency_links.txt +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta.egg-info/entry_points.txt +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/bauta.egg-info/top_level.txt +0 -0
- {bauta-0.2.0 → bauta-0.2.2}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: bauta
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: Safe, realistic copies of production across databases: masking, subsets, synthetic data, and the ETL to move them
|
|
5
5
|
Author: David Ribeiro
|
|
6
6
|
License-Expression: MIT
|
|
@@ -42,17 +42,20 @@ Provides-Extra: mariadb
|
|
|
42
42
|
Requires-Dist: mysql-connector-python>=8.4; extra == "mariadb"
|
|
43
43
|
Provides-Extra: sqlite
|
|
44
44
|
Provides-Extra: duckdb
|
|
45
|
-
Requires-Dist: duckdb>=1.
|
|
46
|
-
Requires-Dist: pyarrow>=
|
|
45
|
+
Requires-Dist: duckdb>=1.4; extra == "duckdb"
|
|
46
|
+
Requires-Dist: pyarrow>=21; extra == "duckdb"
|
|
47
47
|
Requires-Dist: pytz>=2024.1; extra == "duckdb"
|
|
48
48
|
Provides-Extra: files
|
|
49
|
-
Requires-Dist: pyarrow>=
|
|
49
|
+
Requires-Dist: pyarrow>=21; extra == "files"
|
|
50
|
+
Provides-Extra: iceberg
|
|
51
|
+
Requires-Dist: pyiceberg[glue,pyarrow,sql-sqlite]>=0.12; extra == "iceberg"
|
|
52
|
+
Requires-Dist: pyarrow>=21; extra == "iceberg"
|
|
50
53
|
Provides-Extra: fpe
|
|
51
54
|
Requires-Dist: cryptography>=42; extra == "fpe"
|
|
52
55
|
Provides-Extra: all
|
|
53
|
-
Requires-Dist: bauta[duckdb,files,fpe,mariadb,mssql,mysql,oracle,postgresql,sqlite]; extra == "all"
|
|
56
|
+
Requires-Dist: bauta[duckdb,files,fpe,iceberg,mariadb,mssql,mysql,oracle,postgresql,sqlite]; extra == "all"
|
|
54
57
|
Provides-Extra: native
|
|
55
|
-
Requires-Dist: bauta-rs==0.2.
|
|
58
|
+
Requires-Dist: bauta-rs==0.2.2; extra == "native"
|
|
56
59
|
Provides-Extra: dev
|
|
57
60
|
Requires-Dist: pytest==9.1.1; extra == "dev"
|
|
58
61
|
Requires-Dist: mypy==2.3.1; extra == "dev"
|
|
@@ -73,6 +76,7 @@ Dynamic: license-file
|
|
|
73
76
|
- **Provable coverage:** list every table in production and what the jobs do with each, so a table nobody wrote a job for fails the build rather than going unnoticed. A database can require that nothing reaches it unmasked.
|
|
74
77
|
- **Seven databases:** Oracle, SQL Server, PostgreSQL, MySQL, MariaDB, SQLite and DuckDB, as source or target in any combination.
|
|
75
78
|
- **Files as a target:** a masked copy in Parquet, CSV or JSON Lines, in a directory or in S3, Google Cloud Storage or Azure, for a data lake that Athena, Snowflake or Databricks reads: appended to incrementally or published whole as a snapshot, with nothing visible until a run succeeds.
|
|
79
|
+
- **Iceberg tables:** a masked copy as Iceberg tables in any of the three clouds, through a Glue, REST or SQL catalog: each run one commit, upserts that merge by key, and old snapshots' files deleted with them.
|
|
76
80
|
- **Streaming:** memory stays flat however large the table, and PostgreSQL and SQL Server targets load in bulk.
|
|
77
81
|
- **Fast:** a million rows of six masked columns, two of them one-to-one keys, in under 8 seconds on one core with the optional native masker, and 74 without; more cores for wide tables. Either way the masks are the same.
|
|
78
82
|
- **Incremental loads:** extract only what changed since the last successful run.
|
|
@@ -98,11 +102,12 @@ pip install "bauta[postgresql,oracle]"
|
|
|
98
102
|
| `sqlite` | Python's own `sqlite3` | nothing |
|
|
99
103
|
| `duckdb` | duckdb, with pyarrow for fast loads | nothing; one process at a time per file, see [DuckDB](docs/configuration.md#duckdb) |
|
|
100
104
|
| `files` | pyarrow, to write Parquet, CSV and JSON Lines, locally or to S3, GCS or Azure | nothing; see [files](docs/configuration.md#files) |
|
|
105
|
+
| `iceberg` | pyiceberg, with its Glue and SQL catalogs | nothing; see [Iceberg](docs/configuration.md#iceberg) |
|
|
101
106
|
| `fpe` | cryptography, for the `fpe` masking strategy | nothing; `oracle` already brings it |
|
|
102
107
|
| `native` | `bauta-rs`, the native masker (below) | nothing on Linux (x86-64, ARM) or macOS; elsewhere, [Rust](https://rustup.rs) 1.83 or newer |
|
|
103
108
|
| `all` | every driver above | nothing |
|
|
104
109
|
|
|
105
|
-
**The native masker (optional).** `bauta-rs` masks in Rust: about ten times the throughput on one core, with identical masks. It can also mask on several cores: `jobs.yaml`'s `maskingThreads` is `1` by default, a number up to the cores available, or `auto` to divide the cores between the jobs running (see [masking threads](docs/masking.md#masking-threads)). `pip install "bauta[postgresql,native]"` installs the version that matches, which is the only one Bauta uses. Without it, everything works, only slower. See [the native masker](docs/masking.md#the-native-masker).
|
|
110
|
+
**The native masker (optional).** `bauta-rs` masks in Rust: about ten times the throughput on one core, with identical masks. It can also mask on several cores: `jobs.yaml`'s `maskingThreads` is `1` by default, a number up to the cores available, or `auto` to divide half the cores between the jobs running (see [masking threads](docs/masking.md#masking-threads)). `pip install "bauta[postgresql,native]"` installs the version that matches, which is the only one Bauta uses. Without it, everything works, only slower. See [the native masker](docs/masking.md#the-native-masker).
|
|
106
111
|
|
|
107
112
|
|
|
108
113
|
## Quickstart
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
- **Provable coverage:** list every table in production and what the jobs do with each, so a table nobody wrote a job for fails the build rather than going unnoticed. A database can require that nothing reaches it unmasked.
|
|
9
9
|
- **Seven databases:** Oracle, SQL Server, PostgreSQL, MySQL, MariaDB, SQLite and DuckDB, as source or target in any combination.
|
|
10
10
|
- **Files as a target:** a masked copy in Parquet, CSV or JSON Lines, in a directory or in S3, Google Cloud Storage or Azure, for a data lake that Athena, Snowflake or Databricks reads: appended to incrementally or published whole as a snapshot, with nothing visible until a run succeeds.
|
|
11
|
+
- **Iceberg tables:** a masked copy as Iceberg tables in any of the three clouds, through a Glue, REST or SQL catalog: each run one commit, upserts that merge by key, and old snapshots' files deleted with them.
|
|
11
12
|
- **Streaming:** memory stays flat however large the table, and PostgreSQL and SQL Server targets load in bulk.
|
|
12
13
|
- **Fast:** a million rows of six masked columns, two of them one-to-one keys, in under 8 seconds on one core with the optional native masker, and 74 without; more cores for wide tables. Either way the masks are the same.
|
|
13
14
|
- **Incremental loads:** extract only what changed since the last successful run.
|
|
@@ -33,11 +34,12 @@ pip install "bauta[postgresql,oracle]"
|
|
|
33
34
|
| `sqlite` | Python's own `sqlite3` | nothing |
|
|
34
35
|
| `duckdb` | duckdb, with pyarrow for fast loads | nothing; one process at a time per file, see [DuckDB](docs/configuration.md#duckdb) |
|
|
35
36
|
| `files` | pyarrow, to write Parquet, CSV and JSON Lines, locally or to S3, GCS or Azure | nothing; see [files](docs/configuration.md#files) |
|
|
37
|
+
| `iceberg` | pyiceberg, with its Glue and SQL catalogs | nothing; see [Iceberg](docs/configuration.md#iceberg) |
|
|
36
38
|
| `fpe` | cryptography, for the `fpe` masking strategy | nothing; `oracle` already brings it |
|
|
37
39
|
| `native` | `bauta-rs`, the native masker (below) | nothing on Linux (x86-64, ARM) or macOS; elsewhere, [Rust](https://rustup.rs) 1.83 or newer |
|
|
38
40
|
| `all` | every driver above | nothing |
|
|
39
41
|
|
|
40
|
-
**The native masker (optional).** `bauta-rs` masks in Rust: about ten times the throughput on one core, with identical masks. It can also mask on several cores: `jobs.yaml`'s `maskingThreads` is `1` by default, a number up to the cores available, or `auto` to divide the cores between the jobs running (see [masking threads](docs/masking.md#masking-threads)). `pip install "bauta[postgresql,native]"` installs the version that matches, which is the only one Bauta uses. Without it, everything works, only slower. See [the native masker](docs/masking.md#the-native-masker).
|
|
42
|
+
**The native masker (optional).** `bauta-rs` masks in Rust: about ten times the throughput on one core, with identical masks. It can also mask on several cores: `jobs.yaml`'s `maskingThreads` is `1` by default, a number up to the cores available, or `auto` to divide half the cores between the jobs running (see [masking threads](docs/masking.md#masking-threads)). `pip install "bauta[postgresql,native]"` installs the version that matches, which is the only one Bauta uses. Without it, everything works, only slower. See [the native masker](docs/masking.md#the-native-masker).
|
|
41
43
|
|
|
42
44
|
|
|
43
45
|
## Quickstart
|
|
@@ -13,7 +13,7 @@ from typing import Any, Dict, Generator, List, Mapping, Optional, Sequence, Tupl
|
|
|
13
13
|
|
|
14
14
|
import yaml
|
|
15
15
|
|
|
16
|
-
from ..configuration import (Configuration, ConfigurationError, ConnectionConfig, DatabaseConfig, DataJobConfig, DataJobsFile, FilesConnection,
|
|
16
|
+
from ..configuration import (Configuration, ConfigurationError, ConnectionConfig, DatabaseConfig, DataJobConfig, DataJobsFile, FilesConnection, IcebergConnection, isLake,
|
|
17
17
|
StorageLocation, TableLocation, expandEnvironmentVariables)
|
|
18
18
|
from ..database import Database
|
|
19
19
|
from ..log import Log
|
|
@@ -189,9 +189,9 @@ def _loadDataJobs(arguments: argparse.Namespace) -> Tuple[DataJobsFile, Dict[str
|
|
|
189
189
|
|
|
190
190
|
unknown = ['{}: connection "{}" is not a known connection alias'.format(setting, location.connection)
|
|
191
191
|
for setting, location in jobsFile.tableLocations().items() if location.connection not in connectionConfiguration]
|
|
192
|
-
unknown += ['{}: connection "{}" is a files connection; {} is kept in a database table or a file'.format(
|
|
193
|
-
|
|
194
|
-
if
|
|
192
|
+
unknown += ['{}: connection "{}" is a files or Iceberg connection; {} is kept in a database table or a file'.format(
|
|
193
|
+
setting, location.connection, setting)
|
|
194
|
+
for setting, location in jobsFile.tableLocations().items() if isLake(connectionConfiguration.get(location.connection))]
|
|
195
195
|
if unknown:
|
|
196
196
|
raise ConfigurationError('Invalid configuration in {}:\n'.format(jobsPath) + '\n'.join(unknown))
|
|
197
197
|
|
|
@@ -385,8 +385,9 @@ def _requireDatabase(connectionConfiguration: Dict[str, ConnectionConfig], alias
|
|
|
385
385
|
|
|
386
386
|
_requireAlias(connectionConfiguration, alias)
|
|
387
387
|
settings = connectionConfiguration[alias]
|
|
388
|
-
if
|
|
389
|
-
raise UsageError('{} is a
|
|
390
|
-
'targetConnection
|
|
388
|
+
if isLake(settings):
|
|
389
|
+
raise UsageError('{} is a {} connection; this command works with databases. A job writing to it names it as its '
|
|
390
|
+
'targetConnection'.format(alias, settings.type.value))
|
|
391
|
+
assert not isinstance(settings, (FilesConnection, IcebergConnection))
|
|
391
392
|
|
|
392
393
|
return settings
|
|
@@ -7,6 +7,7 @@ import argparse
|
|
|
7
7
|
import datetime
|
|
8
8
|
from typing import Dict, List, Sequence, Tuple
|
|
9
9
|
|
|
10
|
+
from ..configuration import isLake
|
|
10
11
|
from ..database import Database
|
|
11
12
|
from ..database.dialects import catalogTable, quoteIdentifier, quoteTableName
|
|
12
13
|
from ..log import Log
|
|
@@ -296,9 +297,10 @@ def _commandClear(arguments: argparse.Namespace, log: Log) -> int:
|
|
|
296
297
|
raise UsageError('refusing to clear the targets of incremental job(s) {}: their stored watermark would make the next run '
|
|
297
298
|
'load only new rows. Leave them out with --job'.format(', '.join(incremental)))
|
|
298
299
|
|
|
299
|
-
|
|
300
|
-
if
|
|
301
|
-
raise UsageError('clear empties tables in databases, and job(s) {} write files. Leave them out with
|
|
300
|
+
writingLakes = sorted(name for name, job in jobs.items() if isLake(connectionConfiguration.get(job.targetConnection)))
|
|
301
|
+
if writingLakes:
|
|
302
|
+
raise UsageError('clear empties tables in databases, and job(s) {} write files or Iceberg tables. Leave them out with '
|
|
303
|
+
'--job'.format(', '.join(writingLakes)))
|
|
302
304
|
|
|
303
305
|
tablesByDatabase: Dict[str, List[str]] = {}
|
|
304
306
|
for job in jobs.values():
|
|
@@ -7,7 +7,7 @@ import argparse
|
|
|
7
7
|
import json
|
|
8
8
|
from typing import Any, Dict, List, Mapping, Optional
|
|
9
9
|
|
|
10
|
-
from ..configuration import EMBEDDED_TYPES,
|
|
10
|
+
from ..configuration import EMBEDDED_TYPES, isLake
|
|
11
11
|
from ..database import Database
|
|
12
12
|
from ..database.dialects import ForeignKey, bareName, unqualifiedName
|
|
13
13
|
from ..log import Log
|
|
@@ -40,7 +40,7 @@ def _commandVerifyReferences(arguments: argparse.Namespace, log: Log) -> int:
|
|
|
40
40
|
log.logging.warning('{}: could not read foreign keys, so only the target\'s own are checked -- {}'.format(alias, describeError(error)))
|
|
41
41
|
|
|
42
42
|
results = []
|
|
43
|
-
for target in sorted({job.targetConnection for job in jobs.values() if not job.
|
|
43
|
+
for target in sorted({job.targetConnection for job in jobs.values() if not isLake(connectionConfiguration[job.targetConnection])}):
|
|
44
44
|
targetJobs = [job for job in jobs.values() if job.targetConnection == target]
|
|
45
45
|
# Keyed bare, as the catalogs report names: a job naming a reserved
|
|
46
46
|
# word writes it quoted, and a quoted key would match nothing.
|
|
@@ -143,14 +143,15 @@ def _commandAudit(arguments: argparse.Namespace, log: Log) -> int:
|
|
|
143
143
|
try:
|
|
144
144
|
returnedColumns[name] = _sourceQueryColumns(job, connections)
|
|
145
145
|
if job.masking is not None:
|
|
146
|
-
# A file target writes the columns the query returns.
|
|
147
|
-
|
|
146
|
+
# A file or Iceberg target writes the columns the query returns.
|
|
147
|
+
writesLake = isLake(connectionConfiguration[job.targetConnection])
|
|
148
|
+
targetColumns[name] = job.targetColumns or (returnedColumns[name] if writesLake else _targetColumns(job, connections))
|
|
148
149
|
except Exception as error:
|
|
149
150
|
unreachable[name] = describeError(error)
|
|
150
151
|
|
|
151
152
|
for alias in sorted({job.sourceConnection for job in jobs.values()} | {job.targetConnection for job in jobs.values()}):
|
|
152
|
-
if
|
|
153
|
-
#
|
|
153
|
+
if isLake(connectionConfiguration[alias]):
|
|
154
|
+
# Files or Iceberg: no connection to ask of encryption,
|
|
154
155
|
# and no foreign keys.
|
|
155
156
|
continue
|
|
156
157
|
isLocal = connectionConfiguration[alias].type in EMBEDDED_TYPES
|
|
@@ -12,7 +12,7 @@ import sys
|
|
|
12
12
|
from pathlib import Path
|
|
13
13
|
from typing import Any, Callable, Dict, List, Optional, Tuple
|
|
14
14
|
|
|
15
|
-
from ..configuration import ConfigurationError, ConnectionConfig, DataJobsFile, FilesConnection, InsertStrategy
|
|
15
|
+
from ..configuration import ConfigurationError, ConnectionConfig, DataJobConfig, DataJobsFile, FilesConnection, IcebergConnection, InsertStrategy, isLake
|
|
16
16
|
from ..database import DIALECTS
|
|
17
17
|
from ..jobs.dependencyGraph import DependencyGraph
|
|
18
18
|
from ..log import Log
|
|
@@ -255,8 +255,9 @@ def _commandValidate(arguments: argparse.Namespace, log: Log) -> int:
|
|
|
255
255
|
problems.append('{}: {} -> {}'.format(name, column, error))
|
|
256
256
|
|
|
257
257
|
for alias, settings in sorted(connectionConfiguration.items()):
|
|
258
|
-
if
|
|
258
|
+
if isLake(settings):
|
|
259
259
|
continue
|
|
260
|
+
assert not isinstance(settings, (FilesConnection, IcebergConnection))
|
|
260
261
|
try:
|
|
261
262
|
DIALECTS[settings.type].connectArguments(settings, resolvePassword=False)
|
|
262
263
|
except ConfigurationError as error:
|
|
@@ -303,9 +304,44 @@ def _commandValidate(arguments: argparse.Namespace, log: Log) -> int:
|
|
|
303
304
|
return EXIT_SUCCESS
|
|
304
305
|
|
|
305
306
|
|
|
307
|
+
def _checkTargetTables(name: str, job: DataJobConfig, connections: _Connections, log: Log, problems: List[str]) -> Optional[List[str]]:
|
|
308
|
+
"""A database target's columns, having checked it is there, keyed for an
|
|
309
|
+
upsert, and has a stage table holding them all; None, with the problem
|
|
310
|
+
added, when the target can't be read.
|
|
311
|
+
"""
|
|
312
|
+
|
|
313
|
+
try:
|
|
314
|
+
with connections.use(job.targetConnection) as database:
|
|
315
|
+
columns = database.getAllColumnNames(table=job.targetTableFinal)
|
|
316
|
+
log.logging.info('{}: target {} has {} column(s)'.format(name, job.targetTableFinal, len(columns)))
|
|
317
|
+
|
|
318
|
+
if job.insertStrategy == InsertStrategy.UPSERT and not database.getPrimaryColumnNames(table=job.targetTableFinal):
|
|
319
|
+
problems.append('{}: target {} has no primary key, so insertStrategy: upsert cannot match rows'.format(name, job.targetTableFinal))
|
|
320
|
+
except Exception as error:
|
|
321
|
+
problems.append('{}: target {} is not readable -- {}'.format(name, job.targetTableFinal, describeError(error)))
|
|
322
|
+
return None
|
|
323
|
+
|
|
324
|
+
# The stage table is where the rows actually land, so a run fails at
|
|
325
|
+
# once without it.
|
|
326
|
+
if job.targetTableStage:
|
|
327
|
+
try:
|
|
328
|
+
with connections.use(job.targetConnection) as database:
|
|
329
|
+
staged = {column.upper() for column in database.getAllColumnNames(table=job.targetTableStage)}
|
|
330
|
+
log.logging.info('{}: stage table {} has {} column(s)'.format(name, job.targetTableStage, len(staged)))
|
|
331
|
+
missing = [column for column in columns if column.upper() not in staged]
|
|
332
|
+
if missing:
|
|
333
|
+
problems.append('{}: stage table {} is missing column(s) {}, which the load writes'.format(
|
|
334
|
+
name, job.targetTableStage, ', '.join(missing)))
|
|
335
|
+
except Exception as error:
|
|
336
|
+
problems.append('{}: stage table {} is not readable -- {}'.format(name, job.targetTableStage, describeError(error)))
|
|
337
|
+
|
|
338
|
+
return columns
|
|
339
|
+
|
|
340
|
+
|
|
306
341
|
def _dryRunDataJobs(jobsFile: DataJobsFile, connectionConfiguration: Dict[str, ConnectionConfig], log: Log) -> int:
|
|
307
342
|
"""Everything `validate` does, plus what needs a connection: that each alias
|
|
308
|
-
connects, target tables exist,
|
|
343
|
+
connects or can be written, target tables exist, upsert targets have a
|
|
344
|
+
primary key, and each query's columns fit its target and masking policy.
|
|
309
345
|
"""
|
|
310
346
|
|
|
311
347
|
problems: List[str] = []
|
|
@@ -316,8 +352,8 @@ def _dryRunDataJobs(jobsFile: DataJobsFile, connectionConfiguration: Dict[str, C
|
|
|
316
352
|
|
|
317
353
|
for alias in aliases:
|
|
318
354
|
settings = connectionConfiguration[alias]
|
|
319
|
-
if
|
|
320
|
-
from ..
|
|
355
|
+
if isLake(settings):
|
|
356
|
+
from ..lake import checkWritable
|
|
321
357
|
|
|
322
358
|
try:
|
|
323
359
|
checkWritable(settings)
|
|
@@ -338,48 +374,12 @@ def _dryRunDataJobs(jobsFile: DataJobsFile, connectionConfiguration: Dict[str, C
|
|
|
338
374
|
for name, job in jobsFile.jobs.items():
|
|
339
375
|
if job.targetConnection in unreachable:
|
|
340
376
|
continue
|
|
341
|
-
if job.writesFiles():
|
|
342
|
-
# A file target has no table to read yet: it takes the
|
|
343
|
-
# columns the query returns.
|
|
344
|
-
if job.sourceConnection in unreachable:
|
|
345
|
-
continue
|
|
346
|
-
try:
|
|
347
|
-
returned = _sourceQueryColumns(job, connections)
|
|
348
|
-
except Exception as error:
|
|
349
|
-
problems.append('{}: sourceQuery could not be checked -- {}'.format(name, describeError(error)))
|
|
350
|
-
continue
|
|
351
|
-
problem = _checkColumnCounts(name, job, returned, returned)
|
|
352
|
-
if problem is None and job.masking is not None:
|
|
353
|
-
problem = _checkMaskingCoverage(name, job, returned, log)
|
|
354
|
-
if problem:
|
|
355
|
-
problems.append(problem)
|
|
356
|
-
continue
|
|
357
|
-
try:
|
|
358
|
-
with connections.use(job.targetConnection) as database:
|
|
359
|
-
columns = database.getAllColumnNames(table=job.targetTableFinal)
|
|
360
|
-
log.logging.info('{}: target {} has {} column(s)'.format(name, job.targetTableFinal, len(columns)))
|
|
361
377
|
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
# The stage table is where the rows actually land, so a run fails at
|
|
369
|
-
# once without it.
|
|
370
|
-
if job.targetTableStage:
|
|
371
|
-
try:
|
|
372
|
-
with connections.use(job.targetConnection) as database:
|
|
373
|
-
staged = {column.upper() for column in database.getAllColumnNames(table=job.targetTableStage)}
|
|
374
|
-
log.logging.info('{}: stage table {} has {} column(s)'.format(name, job.targetTableStage, len(staged)))
|
|
375
|
-
missing = [column for column in columns if column.upper() not in staged]
|
|
376
|
-
if missing:
|
|
377
|
-
problems.append('{}: stage table {} is missing column(s) {}, which the load writes'.format(
|
|
378
|
-
name, job.targetTableStage, ', '.join(missing)))
|
|
379
|
-
except Exception as error:
|
|
380
|
-
problems.append('{}: stage table {} is not readable -- {}'.format(name, job.targetTableStage, describeError(error)))
|
|
381
|
-
|
|
382
|
-
if job.sourceConnection in unreachable:
|
|
378
|
+
# A lake has no table to read before the first run: it takes the
|
|
379
|
+
# columns the query returns.
|
|
380
|
+
lake = isLake(connectionConfiguration[job.targetConnection])
|
|
381
|
+
columns = None if lake else _checkTargetTables(name, job, connections, log, problems)
|
|
382
|
+
if (not lake and columns is None) or job.sourceConnection in unreachable:
|
|
383
383
|
continue
|
|
384
384
|
|
|
385
385
|
try:
|
|
@@ -391,13 +391,11 @@ def _dryRunDataJobs(jobsFile: DataJobsFile, connectionConfiguration: Dict[str, C
|
|
|
391
391
|
# The same comparison the load makes, made before it writes: a target
|
|
392
392
|
# that gained or lost a column against a query that didn't is the
|
|
393
393
|
# ordinary way a working job stops working.
|
|
394
|
-
problem = _checkColumnCounts(name, job, returned, columns)
|
|
394
|
+
problem = _checkColumnCounts(name, job, returned, returned if columns is None else columns)
|
|
395
|
+
if problem is None and job.masking is not None:
|
|
396
|
+
problem = _checkMaskingCoverage(name, job, returned, log)
|
|
395
397
|
if problem:
|
|
396
398
|
problems.append(problem)
|
|
397
|
-
elif job.masking is not None:
|
|
398
|
-
problem = _checkMaskingCoverage(name, job, returned, log)
|
|
399
|
-
if problem:
|
|
400
|
-
problems.append(problem)
|
|
401
399
|
|
|
402
400
|
if problems:
|
|
403
401
|
for problem in problems:
|
|
@@ -1,14 +1,16 @@
|
|
|
1
1
|
"""Reading and validating jobs.yaml, connections.yaml and discovery.yaml: what
|
|
2
2
|
`environment` resolves from outside the files first, then the pydantic
|
|
3
|
-
models
|
|
3
|
+
models the rest of the package takes: the connections in `connections`, and
|
|
4
|
+
the jobs and discovery rules in `models`.
|
|
4
5
|
"""
|
|
5
6
|
from .environment import ConfigurationError, PasswordCommandError, expandEnvironmentVariables, runPasswordCommand
|
|
6
|
-
from .
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
7
|
+
from .connections import (EMBEDDED_TYPES, IDENTIFIER, S3_LARGEST_COPY, ConnectionConfig, DatabaseConfig, DatabaseType, DuckDBConnection,
|
|
8
|
+
FileCompression, FileFormat, FilesConnection, FileStore, IcebergCatalog, IcebergConnection, LakeConfig, MariaDBConnection,
|
|
9
|
+
MSSQLConnection, MySQLConnection, OracleConnection, PostgreSQLConnection, SQLiteConnection, StoreType)
|
|
10
|
+
from .models import (FILE_STRATEGIES, WATERMARK_PLACEHOLDER, BaseJobConfig, Configuration, DataJobConfig, DataJobsFile, DiscoveryRulesFile,
|
|
11
|
+
InsertStrategy, MaskingConfig, NameRuleConfig, StorageLocation, TableLocation, ValueRuleConfig, connectionConfig, filePathProblem,
|
|
12
|
+
findCycle, isLake, targetKind, targetProblems)
|
|
13
|
+
from .columnTypes import ColumnType, parseColumnType
|
|
12
14
|
|
|
13
15
|
__all__ = [
|
|
14
16
|
'BaseJobConfig',
|
|
@@ -29,6 +31,11 @@ __all__ = [
|
|
|
29
31
|
'FileFormat',
|
|
30
32
|
'filePathProblem',
|
|
31
33
|
'FilesConnection',
|
|
34
|
+
'FileStore',
|
|
35
|
+
'IcebergCatalog',
|
|
36
|
+
'IcebergConnection',
|
|
37
|
+
'isLake',
|
|
38
|
+
'LakeConfig',
|
|
32
39
|
'findCycle',
|
|
33
40
|
'EMBEDDED_TYPES',
|
|
34
41
|
'IDENTIFIER',
|
|
@@ -43,11 +50,13 @@ __all__ = [
|
|
|
43
50
|
'PasswordCommandError',
|
|
44
51
|
'PostgreSQLConnection',
|
|
45
52
|
'runPasswordCommand',
|
|
53
|
+
'S3_LARGEST_COPY',
|
|
46
54
|
'SQLiteConnection',
|
|
47
55
|
'StorageLocation',
|
|
48
56
|
'StoreType',
|
|
49
57
|
'TableLocation',
|
|
50
|
-
'
|
|
58
|
+
'targetKind',
|
|
59
|
+
'targetProblems',
|
|
51
60
|
'ValueRuleConfig',
|
|
52
61
|
'WATERMARK_PLACEHOLDER',
|
|
53
62
|
]
|