bauta 0.2.0__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. {bauta-0.2.0 → bauta-0.2.2}/PKG-INFO +12 -7
  2. {bauta-0.2.0 → bauta-0.2.2}/README.md +3 -1
  3. {bauta-0.2.0 → bauta-0.2.2}/bauta/cli/common.py +8 -7
  4. {bauta-0.2.0 → bauta-0.2.2}/bauta/cli/generate.py +5 -3
  5. {bauta-0.2.0 → bauta-0.2.2}/bauta/cli/review.py +7 -6
  6. {bauta-0.2.0 → bauta-0.2.2}/bauta/cli/run.py +49 -51
  7. {bauta-0.2.0 → bauta-0.2.2}/bauta/configuration/__init__.py +17 -8
  8. bauta-0.2.2/bauta/configuration/connections.py +786 -0
  9. {bauta-0.2.0 → bauta-0.2.2}/bauta/configuration/models.py +132 -631
  10. {bauta-0.2.0 → bauta-0.2.2}/bauta/database/connection.py +23 -9
  11. {bauta-0.2.0 → bauta-0.2.2}/bauta/database/values.py +131 -28
  12. {bauta-0.2.0 → bauta-0.2.2}/bauta/jobs/pipeline.py +25 -21
  13. {bauta-0.2.0 → bauta-0.2.2}/bauta/jobs/runner.py +3 -2
  14. {bauta-0.2.0 → bauta-0.2.2}/bauta/jobs/targets.py +4 -1
  15. bauta-0.2.2/bauta/lake/__init__.py +30 -0
  16. bauta-0.2.2/bauta/lake/columnar.py +204 -0
  17. {bauta-0.2.0/bauta/files → bauta-0.2.2/bauta/lake}/columns.py +21 -8
  18. {bauta-0.2.0/bauta/files → bauta-0.2.2/bauta/lake}/formats.py +135 -15
  19. bauta-0.2.2/bauta/lake/iceberg.py +499 -0
  20. {bauta-0.2.0/bauta/files → bauta-0.2.2/bauta/lake}/stores.py +15 -55
  21. bauta-0.2.2/bauta/lake/target.py +204 -0
  22. {bauta-0.2.0 → bauta-0.2.2}/bauta/masking/core.py +79 -11
  23. {bauta-0.2.0 → bauta-0.2.2}/bauta.egg-info/PKG-INFO +12 -7
  24. {bauta-0.2.0 → bauta-0.2.2}/bauta.egg-info/SOURCES.txt +9 -6
  25. {bauta-0.2.0 → bauta-0.2.2}/bauta.egg-info/requires.txt +9 -5
  26. {bauta-0.2.0 → bauta-0.2.2}/pyproject.toml +11 -6
  27. bauta-0.2.0/bauta/files/__init__.py +0 -11
  28. bauta-0.2.0/bauta/files/target.py +0 -419
  29. {bauta-0.2.0 → bauta-0.2.2}/LICENSE +0 -0
  30. {bauta-0.2.0 → bauta-0.2.2}/bauta/__init__.py +0 -0
  31. {bauta-0.2.0 → bauta-0.2.2}/bauta/cli/__init__.py +0 -0
  32. /bauta-0.2.0/bauta/configuration/fileTypes.py → /bauta-0.2.2/bauta/configuration/columnTypes.py +0 -0
  33. {bauta-0.2.0 → bauta-0.2.2}/bauta/configuration/environment.py +0 -0
  34. {bauta-0.2.0 → bauta-0.2.2}/bauta/database/__init__.py +0 -0
  35. {bauta-0.2.0 → bauta-0.2.2}/bauta/database/dialects/__init__.py +0 -0
  36. {bauta-0.2.0 → bauta-0.2.2}/bauta/database/dialects/base.py +0 -0
  37. {bauta-0.2.0 → bauta-0.2.2}/bauta/database/dialects/duckdb.py +0 -0
  38. {bauta-0.2.0 → bauta-0.2.2}/bauta/database/dialects/mssql.py +0 -0
  39. {bauta-0.2.0 → bauta-0.2.2}/bauta/database/dialects/mysql.py +0 -0
  40. {bauta-0.2.0 → bauta-0.2.2}/bauta/database/dialects/names.py +0 -0
  41. {bauta-0.2.0 → bauta-0.2.2}/bauta/database/dialects/oracle.py +0 -0
  42. {bauta-0.2.0 → bauta-0.2.2}/bauta/database/dialects/postgresql.py +0 -0
  43. {bauta-0.2.0 → bauta-0.2.2}/bauta/database/dialects/sqlite.py +0 -0
  44. {bauta-0.2.0 → bauta-0.2.2}/bauta/database/driver.py +0 -0
  45. {bauta-0.2.0 → bauta-0.2.2}/bauta/generate/__init__.py +0 -0
  46. {bauta-0.2.0 → bauta-0.2.2}/bauta/generate/builtinDiscovery.py +0 -0
  47. {bauta-0.2.0 → bauta-0.2.2}/bauta/generate/discovery.py +0 -0
  48. {bauta-0.2.0 → bauta-0.2.2}/bauta/generate/schema.py +0 -0
  49. {bauta-0.2.0 → bauta-0.2.2}/bauta/generate/subset.py +0 -0
  50. {bauta-0.2.0 → bauta-0.2.2}/bauta/generate/synthesize.py +0 -0
  51. {bauta-0.2.0 → bauta-0.2.2}/bauta/jobs/__init__.py +0 -0
  52. {bauta-0.2.0 → bauta-0.2.2}/bauta/jobs/dependencyGraph.py +0 -0
  53. {bauta-0.2.0 → bauta-0.2.2}/bauta/jobs/keys.py +0 -0
  54. {bauta-0.2.0 → bauta-0.2.2}/bauta/jobs/memory.py +0 -0
  55. {bauta-0.2.0 → bauta-0.2.2}/bauta/jobs/reporting.py +0 -0
  56. {bauta-0.2.0 → bauta-0.2.2}/bauta/jobs/workers.py +0 -0
  57. {bauta-0.2.0 → bauta-0.2.2}/bauta/log/__init__.py +0 -0
  58. {bauta-0.2.0 → bauta-0.2.2}/bauta/log/core.py +0 -0
  59. {bauta-0.2.0 → bauta-0.2.2}/bauta/log/scrubbing.py +0 -0
  60. {bauta-0.2.0 → bauta-0.2.2}/bauta/masking/__init__.py +0 -0
  61. {bauta-0.2.0 → bauta-0.2.2}/bauta/masking/fakeData.py +0 -0
  62. {bauta-0.2.0 → bauta-0.2.2}/bauta/masking/fpe.py +0 -0
  63. {bauta-0.2.0 → bauta-0.2.2}/bauta/masking/strategies.py +0 -0
  64. {bauta-0.2.0 → bauta-0.2.2}/bauta/py.typed +0 -0
  65. {bauta-0.2.0 → bauta-0.2.2}/bauta/review/__init__.py +0 -0
  66. {bauta-0.2.0 → bauta-0.2.2}/bauta/review/audit.py +0 -0
  67. {bauta-0.2.0 → bauta-0.2.2}/bauta/review/coverage.py +0 -0
  68. {bauta-0.2.0 → bauta-0.2.2}/bauta/review/references.py +0 -0
  69. {bauta-0.2.0 → bauta-0.2.2}/bauta/transform/__init__.py +0 -0
  70. {bauta-0.2.0 → bauta-0.2.2}/bauta/transform/builtinTransforms.py +0 -0
  71. {bauta-0.2.0 → bauta-0.2.2}/bauta/transform/core.py +0 -0
  72. {bauta-0.2.0 → bauta-0.2.2}/bauta.egg-info/dependency_links.txt +0 -0
  73. {bauta-0.2.0 → bauta-0.2.2}/bauta.egg-info/entry_points.txt +0 -0
  74. {bauta-0.2.0 → bauta-0.2.2}/bauta.egg-info/top_level.txt +0 -0
  75. {bauta-0.2.0 → bauta-0.2.2}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: bauta
3
- Version: 0.2.0
3
+ Version: 0.2.2
4
4
  Summary: Safe, realistic copies of production across databases: masking, subsets, synthetic data, and the ETL to move them
5
5
  Author: David Ribeiro
6
6
  License-Expression: MIT
@@ -42,17 +42,20 @@ Provides-Extra: mariadb
42
42
  Requires-Dist: mysql-connector-python>=8.4; extra == "mariadb"
43
43
  Provides-Extra: sqlite
44
44
  Provides-Extra: duckdb
45
- Requires-Dist: duckdb>=1.2; extra == "duckdb"
46
- Requires-Dist: pyarrow>=18; extra == "duckdb"
45
+ Requires-Dist: duckdb>=1.4; extra == "duckdb"
46
+ Requires-Dist: pyarrow>=21; extra == "duckdb"
47
47
  Requires-Dist: pytz>=2024.1; extra == "duckdb"
48
48
  Provides-Extra: files
49
- Requires-Dist: pyarrow>=18; extra == "files"
49
+ Requires-Dist: pyarrow>=21; extra == "files"
50
+ Provides-Extra: iceberg
51
+ Requires-Dist: pyiceberg[glue,pyarrow,sql-sqlite]>=0.12; extra == "iceberg"
52
+ Requires-Dist: pyarrow>=21; extra == "iceberg"
50
53
  Provides-Extra: fpe
51
54
  Requires-Dist: cryptography>=42; extra == "fpe"
52
55
  Provides-Extra: all
53
- Requires-Dist: bauta[duckdb,files,fpe,mariadb,mssql,mysql,oracle,postgresql,sqlite]; extra == "all"
56
+ Requires-Dist: bauta[duckdb,files,fpe,iceberg,mariadb,mssql,mysql,oracle,postgresql,sqlite]; extra == "all"
54
57
  Provides-Extra: native
55
- Requires-Dist: bauta-rs==0.2.0; extra == "native"
58
+ Requires-Dist: bauta-rs==0.2.2; extra == "native"
56
59
  Provides-Extra: dev
57
60
  Requires-Dist: pytest==9.1.1; extra == "dev"
58
61
  Requires-Dist: mypy==2.3.1; extra == "dev"
@@ -73,6 +76,7 @@ Dynamic: license-file
73
76
  - **Provable coverage:** list every table in production and what the jobs do with each, so a table nobody wrote a job for fails the build rather than going unnoticed. A database can require that nothing reaches it unmasked.
74
77
  - **Seven databases:** Oracle, SQL Server, PostgreSQL, MySQL, MariaDB, SQLite and DuckDB, as source or target in any combination.
75
78
  - **Files as a target:** a masked copy in Parquet, CSV or JSON Lines, in a directory or in S3, Google Cloud Storage or Azure, for a data lake that Athena, Snowflake or Databricks reads: appended to incrementally or published whole as a snapshot, with nothing visible until a run succeeds.
79
+ - **Iceberg tables:** a masked copy as Iceberg tables in any of the three clouds, through a Glue, REST or SQL catalog: each run one commit, upserts that merge by key, and old snapshots' files deleted with them.
76
80
  - **Streaming:** memory stays flat however large the table, and PostgreSQL and SQL Server targets load in bulk.
77
81
  - **Fast:** a million rows of six masked columns, two of them one-to-one keys, in under 8 seconds on one core with the optional native masker, and 74 without; more cores for wide tables. Either way the masks are the same.
78
82
  - **Incremental loads:** extract only what changed since the last successful run.
@@ -98,11 +102,12 @@ pip install "bauta[postgresql,oracle]"
98
102
  | `sqlite` | Python's own `sqlite3` | nothing |
99
103
  | `duckdb` | duckdb, with pyarrow for fast loads | nothing; one process at a time per file, see [DuckDB](docs/configuration.md#duckdb) |
100
104
  | `files` | pyarrow, to write Parquet, CSV and JSON Lines, locally or to S3, GCS or Azure | nothing; see [files](docs/configuration.md#files) |
105
+ | `iceberg` | pyiceberg, with its Glue and SQL catalogs | nothing; see [Iceberg](docs/configuration.md#iceberg) |
101
106
  | `fpe` | cryptography, for the `fpe` masking strategy | nothing; `oracle` already brings it |
102
107
  | `native` | `bauta-rs`, the native masker (below) | nothing on Linux (x86-64, ARM) or macOS; elsewhere, [Rust](https://rustup.rs) 1.83 or newer |
103
108
  | `all` | every driver above | nothing |
104
109
 
105
- **The native masker (optional).** `bauta-rs` masks in Rust: about ten times the throughput on one core, with identical masks. It can also mask on several cores: `jobs.yaml`'s `maskingThreads` is `1` by default, a number up to the cores available, or `auto` to divide the cores between the jobs running (see [masking threads](docs/masking.md#masking-threads)). `pip install "bauta[postgresql,native]"` installs the version that matches, which is the only one Bauta uses. Without it, everything works, only slower. See [the native masker](docs/masking.md#the-native-masker).
110
+ **The native masker (optional).** `bauta-rs` masks in Rust: about ten times the throughput on one core, with identical masks. It can also mask on several cores: `jobs.yaml`'s `maskingThreads` is `1` by default, a number up to the cores available, or `auto` to divide half the cores between the jobs running (see [masking threads](docs/masking.md#masking-threads)). `pip install "bauta[postgresql,native]"` installs the version that matches, which is the only one Bauta uses. Without it, everything works, only slower. See [the native masker](docs/masking.md#the-native-masker).
106
111
 
107
112
 
108
113
  ## Quickstart
@@ -8,6 +8,7 @@
8
8
  - **Provable coverage:** list every table in production and what the jobs do with each, so a table nobody wrote a job for fails the build rather than going unnoticed. A database can require that nothing reaches it unmasked.
9
9
  - **Seven databases:** Oracle, SQL Server, PostgreSQL, MySQL, MariaDB, SQLite and DuckDB, as source or target in any combination.
10
10
  - **Files as a target:** a masked copy in Parquet, CSV or JSON Lines, in a directory or in S3, Google Cloud Storage or Azure, for a data lake that Athena, Snowflake or Databricks reads: appended to incrementally or published whole as a snapshot, with nothing visible until a run succeeds.
11
+ - **Iceberg tables:** a masked copy as Iceberg tables in any of the three clouds, through a Glue, REST or SQL catalog: each run one commit, upserts that merge by key, and old snapshots' files deleted with them.
11
12
  - **Streaming:** memory stays flat however large the table, and PostgreSQL and SQL Server targets load in bulk.
12
13
  - **Fast:** a million rows of six masked columns, two of them one-to-one keys, in under 8 seconds on one core with the optional native masker, and 74 without; more cores for wide tables. Either way the masks are the same.
13
14
  - **Incremental loads:** extract only what changed since the last successful run.
@@ -33,11 +34,12 @@ pip install "bauta[postgresql,oracle]"
33
34
  | `sqlite` | Python's own `sqlite3` | nothing |
34
35
  | `duckdb` | duckdb, with pyarrow for fast loads | nothing; one process at a time per file, see [DuckDB](docs/configuration.md#duckdb) |
35
36
  | `files` | pyarrow, to write Parquet, CSV and JSON Lines, locally or to S3, GCS or Azure | nothing; see [files](docs/configuration.md#files) |
37
+ | `iceberg` | pyiceberg, with its Glue and SQL catalogs | nothing; see [Iceberg](docs/configuration.md#iceberg) |
36
38
  | `fpe` | cryptography, for the `fpe` masking strategy | nothing; `oracle` already brings it |
37
39
  | `native` | `bauta-rs`, the native masker (below) | nothing on Linux (x86-64, ARM) or macOS; elsewhere, [Rust](https://rustup.rs) 1.83 or newer |
38
40
  | `all` | every driver above | nothing |
39
41
 
40
- **The native masker (optional).** `bauta-rs` masks in Rust: about ten times the throughput on one core, with identical masks. It can also mask on several cores: `jobs.yaml`'s `maskingThreads` is `1` by default, a number up to the cores available, or `auto` to divide the cores between the jobs running (see [masking threads](docs/masking.md#masking-threads)). `pip install "bauta[postgresql,native]"` installs the version that matches, which is the only one Bauta uses. Without it, everything works, only slower. See [the native masker](docs/masking.md#the-native-masker).
42
+ **The native masker (optional).** `bauta-rs` masks in Rust: about ten times the throughput on one core, with identical masks. It can also mask on several cores: `jobs.yaml`'s `maskingThreads` is `1` by default, a number up to the cores available, or `auto` to divide half the cores between the jobs running (see [masking threads](docs/masking.md#masking-threads)). `pip install "bauta[postgresql,native]"` installs the version that matches, which is the only one Bauta uses. Without it, everything works, only slower. See [the native masker](docs/masking.md#the-native-masker).
41
43
 
42
44
 
43
45
  ## Quickstart
@@ -13,7 +13,7 @@ from typing import Any, Dict, Generator, List, Mapping, Optional, Sequence, Tupl
13
13
 
14
14
  import yaml
15
15
 
16
- from ..configuration import (Configuration, ConfigurationError, ConnectionConfig, DatabaseConfig, DataJobConfig, DataJobsFile, FilesConnection,
16
+ from ..configuration import (Configuration, ConfigurationError, ConnectionConfig, DatabaseConfig, DataJobConfig, DataJobsFile, FilesConnection, IcebergConnection, isLake,
17
17
  StorageLocation, TableLocation, expandEnvironmentVariables)
18
18
  from ..database import Database
19
19
  from ..log import Log
@@ -189,9 +189,9 @@ def _loadDataJobs(arguments: argparse.Namespace) -> Tuple[DataJobsFile, Dict[str
189
189
 
190
190
  unknown = ['{}: connection "{}" is not a known connection alias'.format(setting, location.connection)
191
191
  for setting, location in jobsFile.tableLocations().items() if location.connection not in connectionConfiguration]
192
- unknown += ['{}: connection "{}" is a files connection; {} is kept in a database table or a file'.format(setting, location.connection, setting)
193
- for setting, location in jobsFile.tableLocations().items()
194
- if isinstance(connectionConfiguration.get(location.connection), FilesConnection)]
192
+ unknown += ['{}: connection "{}" is a files or Iceberg connection; {} is kept in a database table or a file'.format(
193
+ setting, location.connection, setting)
194
+ for setting, location in jobsFile.tableLocations().items() if isLake(connectionConfiguration.get(location.connection))]
195
195
  if unknown:
196
196
  raise ConfigurationError('Invalid configuration in {}:\n'.format(jobsPath) + '\n'.join(unknown))
197
197
 
@@ -385,8 +385,9 @@ def _requireDatabase(connectionConfiguration: Dict[str, ConnectionConfig], alias
385
385
 
386
386
  _requireAlias(connectionConfiguration, alias)
387
387
  settings = connectionConfiguration[alias]
388
- if isinstance(settings, FilesConnection):
389
- raise UsageError('{} is a files connection; this command works with databases. A job writing to it names it as its '
390
- 'targetConnection, with insertStrategy append or overwrite'.format(alias))
388
+ if isLake(settings):
389
+ raise UsageError('{} is a {} connection; this command works with databases. A job writing to it names it as its '
390
+ 'targetConnection'.format(alias, settings.type.value))
391
+ assert not isinstance(settings, (FilesConnection, IcebergConnection))
391
392
 
392
393
  return settings
@@ -7,6 +7,7 @@ import argparse
7
7
  import datetime
8
8
  from typing import Dict, List, Sequence, Tuple
9
9
 
10
+ from ..configuration import isLake
10
11
  from ..database import Database
11
12
  from ..database.dialects import catalogTable, quoteIdentifier, quoteTableName
12
13
  from ..log import Log
@@ -296,9 +297,10 @@ def _commandClear(arguments: argparse.Namespace, log: Log) -> int:
296
297
  raise UsageError('refusing to clear the targets of incremental job(s) {}: their stored watermark would make the next run '
297
298
  'load only new rows. Leave them out with --job'.format(', '.join(incremental)))
298
299
 
299
- writingFiles = sorted(name for name, job in jobs.items() if job.writesFiles())
300
- if writingFiles:
301
- raise UsageError('clear empties tables in databases, and job(s) {} write files. Leave them out with --job'.format(', '.join(writingFiles)))
300
+ writingLakes = sorted(name for name, job in jobs.items() if isLake(connectionConfiguration.get(job.targetConnection)))
301
+ if writingLakes:
302
+ raise UsageError('clear empties tables in databases, and job(s) {} write files or Iceberg tables. Leave them out with '
303
+ '--job'.format(', '.join(writingLakes)))
302
304
 
303
305
  tablesByDatabase: Dict[str, List[str]] = {}
304
306
  for job in jobs.values():
@@ -7,7 +7,7 @@ import argparse
7
7
  import json
8
8
  from typing import Any, Dict, List, Mapping, Optional
9
9
 
10
- from ..configuration import EMBEDDED_TYPES, FilesConnection
10
+ from ..configuration import EMBEDDED_TYPES, isLake
11
11
  from ..database import Database
12
12
  from ..database.dialects import ForeignKey, bareName, unqualifiedName
13
13
  from ..log import Log
@@ -40,7 +40,7 @@ def _commandVerifyReferences(arguments: argparse.Namespace, log: Log) -> int:
40
40
  log.logging.warning('{}: could not read foreign keys, so only the target\'s own are checked -- {}'.format(alias, describeError(error)))
41
41
 
42
42
  results = []
43
- for target in sorted({job.targetConnection for job in jobs.values() if not job.writesFiles()}):
43
+ for target in sorted({job.targetConnection for job in jobs.values() if not isLake(connectionConfiguration[job.targetConnection])}):
44
44
  targetJobs = [job for job in jobs.values() if job.targetConnection == target]
45
45
  # Keyed bare, as the catalogs report names: a job naming a reserved
46
46
  # word writes it quoted, and a quoted key would match nothing.
@@ -143,14 +143,15 @@ def _commandAudit(arguments: argparse.Namespace, log: Log) -> int:
143
143
  try:
144
144
  returnedColumns[name] = _sourceQueryColumns(job, connections)
145
145
  if job.masking is not None:
146
- # A file target writes the columns the query returns.
147
- targetColumns[name] = job.targetColumns or (returnedColumns[name] if job.writesFiles() else _targetColumns(job, connections))
146
+ # A file or Iceberg target writes the columns the query returns.
147
+ writesLake = isLake(connectionConfiguration[job.targetConnection])
148
+ targetColumns[name] = job.targetColumns or (returnedColumns[name] if writesLake else _targetColumns(job, connections))
148
149
  except Exception as error:
149
150
  unreachable[name] = describeError(error)
150
151
 
151
152
  for alias in sorted({job.sourceConnection for job in jobs.values()} | {job.targetConnection for job in jobs.values()}):
152
- if isinstance(connectionConfiguration[alias], FilesConnection):
153
- # A directory on this machine: no connection to encrypt,
153
+ if isLake(connectionConfiguration[alias]):
154
+ # Files or Iceberg: no connection to ask of encryption,
154
155
  # and no foreign keys.
155
156
  continue
156
157
  isLocal = connectionConfiguration[alias].type in EMBEDDED_TYPES
@@ -12,7 +12,7 @@ import sys
12
12
  from pathlib import Path
13
13
  from typing import Any, Callable, Dict, List, Optional, Tuple
14
14
 
15
- from ..configuration import ConfigurationError, ConnectionConfig, DataJobsFile, FilesConnection, InsertStrategy
15
+ from ..configuration import ConfigurationError, ConnectionConfig, DataJobConfig, DataJobsFile, FilesConnection, IcebergConnection, InsertStrategy, isLake
16
16
  from ..database import DIALECTS
17
17
  from ..jobs.dependencyGraph import DependencyGraph
18
18
  from ..log import Log
@@ -255,8 +255,9 @@ def _commandValidate(arguments: argparse.Namespace, log: Log) -> int:
255
255
  problems.append('{}: {} -> {}'.format(name, column, error))
256
256
 
257
257
  for alias, settings in sorted(connectionConfiguration.items()):
258
- if isinstance(settings, FilesConnection):
258
+ if isLake(settings):
259
259
  continue
260
+ assert not isinstance(settings, (FilesConnection, IcebergConnection))
260
261
  try:
261
262
  DIALECTS[settings.type].connectArguments(settings, resolvePassword=False)
262
263
  except ConfigurationError as error:
@@ -303,9 +304,44 @@ def _commandValidate(arguments: argparse.Namespace, log: Log) -> int:
303
304
  return EXIT_SUCCESS
304
305
 
305
306
 
307
+ def _checkTargetTables(name: str, job: DataJobConfig, connections: _Connections, log: Log, problems: List[str]) -> Optional[List[str]]:
308
+ """A database target's columns, having checked it is there, keyed for an
309
+ upsert, and has a stage table holding them all; None, with the problem
310
+ added, when the target can't be read.
311
+ """
312
+
313
+ try:
314
+ with connections.use(job.targetConnection) as database:
315
+ columns = database.getAllColumnNames(table=job.targetTableFinal)
316
+ log.logging.info('{}: target {} has {} column(s)'.format(name, job.targetTableFinal, len(columns)))
317
+
318
+ if job.insertStrategy == InsertStrategy.UPSERT and not database.getPrimaryColumnNames(table=job.targetTableFinal):
319
+ problems.append('{}: target {} has no primary key, so insertStrategy: upsert cannot match rows'.format(name, job.targetTableFinal))
320
+ except Exception as error:
321
+ problems.append('{}: target {} is not readable -- {}'.format(name, job.targetTableFinal, describeError(error)))
322
+ return None
323
+
324
+ # The stage table is where the rows actually land, so a run fails at
325
+ # once without it.
326
+ if job.targetTableStage:
327
+ try:
328
+ with connections.use(job.targetConnection) as database:
329
+ staged = {column.upper() for column in database.getAllColumnNames(table=job.targetTableStage)}
330
+ log.logging.info('{}: stage table {} has {} column(s)'.format(name, job.targetTableStage, len(staged)))
331
+ missing = [column for column in columns if column.upper() not in staged]
332
+ if missing:
333
+ problems.append('{}: stage table {} is missing column(s) {}, which the load writes'.format(
334
+ name, job.targetTableStage, ', '.join(missing)))
335
+ except Exception as error:
336
+ problems.append('{}: stage table {} is not readable -- {}'.format(name, job.targetTableStage, describeError(error)))
337
+
338
+ return columns
339
+
340
+
306
341
  def _dryRunDataJobs(jobsFile: DataJobsFile, connectionConfiguration: Dict[str, ConnectionConfig], log: Log) -> int:
307
342
  """Everything `validate` does, plus what needs a connection: that each alias
308
- connects, target tables exist, and upsert targets have a primary key.
343
+ connects or can be written, target tables exist, upsert targets have a
344
+ primary key, and each query's columns fit its target and masking policy.
309
345
  """
310
346
 
311
347
  problems: List[str] = []
@@ -316,8 +352,8 @@ def _dryRunDataJobs(jobsFile: DataJobsFile, connectionConfiguration: Dict[str, C
316
352
 
317
353
  for alias in aliases:
318
354
  settings = connectionConfiguration[alias]
319
- if isinstance(settings, FilesConnection):
320
- from ..files import checkWritable
355
+ if isLake(settings):
356
+ from ..lake import checkWritable
321
357
 
322
358
  try:
323
359
  checkWritable(settings)
@@ -338,48 +374,12 @@ def _dryRunDataJobs(jobsFile: DataJobsFile, connectionConfiguration: Dict[str, C
338
374
  for name, job in jobsFile.jobs.items():
339
375
  if job.targetConnection in unreachable:
340
376
  continue
341
- if job.writesFiles():
342
- # A file target has no table to read yet: it takes the
343
- # columns the query returns.
344
- if job.sourceConnection in unreachable:
345
- continue
346
- try:
347
- returned = _sourceQueryColumns(job, connections)
348
- except Exception as error:
349
- problems.append('{}: sourceQuery could not be checked -- {}'.format(name, describeError(error)))
350
- continue
351
- problem = _checkColumnCounts(name, job, returned, returned)
352
- if problem is None and job.masking is not None:
353
- problem = _checkMaskingCoverage(name, job, returned, log)
354
- if problem:
355
- problems.append(problem)
356
- continue
357
- try:
358
- with connections.use(job.targetConnection) as database:
359
- columns = database.getAllColumnNames(table=job.targetTableFinal)
360
- log.logging.info('{}: target {} has {} column(s)'.format(name, job.targetTableFinal, len(columns)))
361
377
 
362
- if job.insertStrategy == InsertStrategy.UPSERT and not database.getPrimaryColumnNames(table=job.targetTableFinal):
363
- problems.append('{}: target {} has no primary key, so insertStrategy: upsert cannot match rows'.format(name, job.targetTableFinal))
364
- except Exception as error:
365
- problems.append('{}: target {} is not readable -- {}'.format(name, job.targetTableFinal, describeError(error)))
366
- continue
367
-
368
- # The stage table is where the rows actually land, so a run fails at
369
- # once without it.
370
- if job.targetTableStage:
371
- try:
372
- with connections.use(job.targetConnection) as database:
373
- staged = {column.upper() for column in database.getAllColumnNames(table=job.targetTableStage)}
374
- log.logging.info('{}: stage table {} has {} column(s)'.format(name, job.targetTableStage, len(staged)))
375
- missing = [column for column in columns if column.upper() not in staged]
376
- if missing:
377
- problems.append('{}: stage table {} is missing column(s) {}, which the load writes'.format(
378
- name, job.targetTableStage, ', '.join(missing)))
379
- except Exception as error:
380
- problems.append('{}: stage table {} is not readable -- {}'.format(name, job.targetTableStage, describeError(error)))
381
-
382
- if job.sourceConnection in unreachable:
378
+ # A lake has no table to read before the first run: it takes the
379
+ # columns the query returns.
380
+ lake = isLake(connectionConfiguration[job.targetConnection])
381
+ columns = None if lake else _checkTargetTables(name, job, connections, log, problems)
382
+ if (not lake and columns is None) or job.sourceConnection in unreachable:
383
383
  continue
384
384
 
385
385
  try:
@@ -391,13 +391,11 @@ def _dryRunDataJobs(jobsFile: DataJobsFile, connectionConfiguration: Dict[str, C
391
391
  # The same comparison the load makes, made before it writes: a target
392
392
  # that gained or lost a column against a query that didn't is the
393
393
  # ordinary way a working job stops working.
394
- problem = _checkColumnCounts(name, job, returned, columns)
394
+ problem = _checkColumnCounts(name, job, returned, returned if columns is None else columns)
395
+ if problem is None and job.masking is not None:
396
+ problem = _checkMaskingCoverage(name, job, returned, log)
395
397
  if problem:
396
398
  problems.append(problem)
397
- elif job.masking is not None:
398
- problem = _checkMaskingCoverage(name, job, returned, log)
399
- if problem:
400
- problems.append(problem)
401
399
 
402
400
  if problems:
403
401
  for problem in problems:
@@ -1,14 +1,16 @@
1
1
  """Reading and validating jobs.yaml, connections.yaml and discovery.yaml: what
2
2
  `environment` resolves from outside the files first, then the pydantic
3
- models in `models` that the rest of the package takes.
3
+ models the rest of the package takes: the connections in `connections`, and
4
+ the jobs and discovery rules in `models`.
4
5
  """
5
6
  from .environment import ConfigurationError, PasswordCommandError, expandEnvironmentVariables, runPasswordCommand
6
- from .models import (EMBEDDED_TYPES, IDENTIFIER, WATERMARK_PLACEHOLDER, BaseJobConfig, Configuration, ConnectionConfig, DatabaseType, DataJobConfig,
7
- DatabaseConfig, DataJobsFile, DiscoveryRulesFile, DuckDBConnection, FILE_STRATEGIES, FileCompression, FileFormat,
8
- FilesConnection, InsertStrategy, MariaDBConnection, MaskingConfig, MSSQLConnection,
9
- MySQLConnection, NameRuleConfig, OracleConnection, PostgreSQLConnection, SQLiteConnection, StorageLocation, TableLocation,
10
- StoreType, ValueRuleConfig, connectionConfig, filePathProblem, findCycle, targetMismatch)
11
- from .fileTypes import ColumnType, parseColumnType
7
+ from .connections import (EMBEDDED_TYPES, IDENTIFIER, S3_LARGEST_COPY, ConnectionConfig, DatabaseConfig, DatabaseType, DuckDBConnection,
8
+ FileCompression, FileFormat, FilesConnection, FileStore, IcebergCatalog, IcebergConnection, LakeConfig, MariaDBConnection,
9
+ MSSQLConnection, MySQLConnection, OracleConnection, PostgreSQLConnection, SQLiteConnection, StoreType)
10
+ from .models import (FILE_STRATEGIES, WATERMARK_PLACEHOLDER, BaseJobConfig, Configuration, DataJobConfig, DataJobsFile, DiscoveryRulesFile,
11
+ InsertStrategy, MaskingConfig, NameRuleConfig, StorageLocation, TableLocation, ValueRuleConfig, connectionConfig, filePathProblem,
12
+ findCycle, isLake, targetKind, targetProblems)
13
+ from .columnTypes import ColumnType, parseColumnType
12
14
 
13
15
  __all__ = [
14
16
  'BaseJobConfig',
@@ -29,6 +31,11 @@ __all__ = [
29
31
  'FileFormat',
30
32
  'filePathProblem',
31
33
  'FilesConnection',
34
+ 'FileStore',
35
+ 'IcebergCatalog',
36
+ 'IcebergConnection',
37
+ 'isLake',
38
+ 'LakeConfig',
32
39
  'findCycle',
33
40
  'EMBEDDED_TYPES',
34
41
  'IDENTIFIER',
@@ -43,11 +50,13 @@ __all__ = [
43
50
  'PasswordCommandError',
44
51
  'PostgreSQLConnection',
45
52
  'runPasswordCommand',
53
+ 'S3_LARGEST_COPY',
46
54
  'SQLiteConnection',
47
55
  'StorageLocation',
48
56
  'StoreType',
49
57
  'TableLocation',
50
- 'targetMismatch',
58
+ 'targetKind',
59
+ 'targetProblems',
51
60
  'ValueRuleConfig',
52
61
  'WATERMARK_PLACEHOLDER',
53
62
  ]