dataplat 0.2.1__tar.gz → 0.2.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dataplat-0.2.1 → dataplat-0.2.3}/CHANGELOG.md +98 -0
- dataplat-0.2.3/CONTRIBUTING.md +177 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/PKG-INFO +11 -1
- {dataplat-0.2.1 → dataplat-0.2.3}/README.md +10 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/db/dbt_orphans.py +63 -15
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/db/role.py +31 -4
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/services/db/describe.py +72 -8
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/services/db/long_queries.py +29 -2
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/services/db/orphans.py +75 -6
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/services/db/role.py +74 -7
- {dataplat-0.2.1 → dataplat-0.2.3}/pyproject.toml +8 -1
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/cli/test_role.py +5 -0
- dataplat-0.2.3/tests/integration/redshift/__init__.py +19 -0
- dataplat-0.2.3/tests/integration/redshift/conftest.py +1191 -0
- dataplat-0.2.3/tests/integration/redshift/test_conformance.py +281 -0
- dataplat-0.2.3/tests/integration/redshift/test_harness.py +415 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/integration/test_describe_pg.py +84 -27
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/integration/test_long_queries_pg.py +219 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/integration/test_orphans_pg.py +146 -16
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/integration/test_roles_pg.py +51 -24
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/services/db/test_describe.py +62 -1
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/services/db/test_long_queries.py +94 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/services/db/test_orphans.py +49 -3
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/services/db/test_role.py +110 -19
- {dataplat-0.2.1 → dataplat-0.2.3}/uv.lock +1 -1
- {dataplat-0.2.1 → dataplat-0.2.3}/.github/workflows/ci.yml +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/.github/workflows/release.yml +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/.gitignore +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/.python-version +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/LICENSE +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/__init__.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/__init__.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/_lazy.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/_missing.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/_options.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/_prompt.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/_render.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/bi/__init__.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/bi/app.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/bi/superset.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/ci/__init__.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/ci/app.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/ci/github/__init__.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/ci/github/app.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/ci/github/runner.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/cloud/__init__.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/cloud/app.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/cloud/aws/__init__.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/cloud/aws/_common.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/cloud/aws/app.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/cloud/aws/rds.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/cloud/aws/redshift.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/cloud/aws/secrets.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/config.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/db/__init__.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/db/_common.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/db/_report.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/db/describe.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/db/long_queries.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/db/role_create.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/db/role_drop.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/db/role_list.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/db/top_tables.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/ingest/__init__.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/ingest/airbyte/__init__.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/ingest/airbyte/_common.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/ingest/airbyte/_cursor.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/ingest/airbyte/_resource.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/ingest/airbyte/app.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/ingest/airbyte/connections.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/ingest/airbyte/definitions.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/ingest/airbyte/destinations.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/ingest/airbyte/enums.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/ingest/airbyte/jobs.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/ingest/airbyte/sources.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/ingest/airbyte/tags.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/ingest/airbyte/templates.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/ingest/airbyte/tui.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/ingest/airbyte/workspaces.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/ingest/app.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/open.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/cli/status.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/core/__init__.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/core/deps.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/core/envrc.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/core/errors.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/core/registry.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/main.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/services/__init__.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/services/airbyte/__init__.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/services/airbyte/_resource.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/services/airbyte/client.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/services/airbyte/connections.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/services/airbyte/definitions.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/services/airbyte/destinations.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/services/airbyte/jobs.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/services/airbyte/sources.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/services/airbyte/tags.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/services/airbyte/workspaces.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/services/aws/__init__.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/services/aws/auth.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/services/db/__init__.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/services/db/_like.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/services/db/connection.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/services/db/role_admin.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/services/db/role_dialects.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/services/db/targets.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/services/db/top_tables.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/services/superset/__init__.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/dataplat/services/superset/client.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/__init__.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/cli/__init__.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/cli/test_airbyte_commands.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/cli/test_airbyte_cursor_logic.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/cli/test_airbyte_guards.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/cli/test_airbyte_tui.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/cli/test_aws_secrets.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/cli/test_aws_secrets_write.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/cli/test_cli_smoke.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/cli/test_config.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/cli/test_db_common.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/cli/test_db_long_queries.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/cli/test_db_query.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/cli/test_dbt_orphans.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/cli/test_describe.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/cli/test_github_runner.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/cli/test_missing_deps.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/cli/test_open.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/cli/test_prompt.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/cli/test_rds.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/cli/test_redshift.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/cli/test_regression.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/cli/test_render.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/cli/test_role_create.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/cli/test_role_drop.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/cli/test_status.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/cli/test_superset.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/cli/test_top_tables.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/conftest.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/core/__init__.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/core/test_deps.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/core/test_envrc.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/core/test_registry.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/integration/__init__.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/integration/conftest.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/integration/test_harness.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/integration/test_top_tables_pg.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/services/__init__.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/services/airbyte/__init__.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/services/airbyte/test_client.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/services/airbyte/test_connections.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/services/airbyte/test_definitions.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/services/airbyte/test_destinations.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/services/airbyte/test_jobs.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/services/airbyte/test_sources.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/services/airbyte/test_workspaces.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/services/aws/__init__.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/services/aws/test_auth.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/services/db/__init__.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/services/db/test_connection.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/services/db/test_role_admin.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/services/db/test_role_dialects.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/services/db/test_targets.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/services/db/test_top_tables.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/services/superset/__init__.py +0 -0
- {dataplat-0.2.1 → dataplat-0.2.3}/tests/services/superset/test_client.py +0 -0
|
@@ -1,5 +1,103 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.2.3
|
|
4
|
+
|
|
5
|
+
Redshift-only fixes. Nothing changes for PostgreSQL targets.
|
|
6
|
+
|
|
7
|
+
### Fixed
|
|
8
|
+
|
|
9
|
+
- `dp db role show` no longer claims every Redshift user has no password. The
|
|
10
|
+
attribute query reported `password_set=False` unconditionally, but
|
|
11
|
+
`pg_user.passwd` is masked to `'********'` there just as `pg_roles.rolpassword`
|
|
12
|
+
is on PostgreSQL — so it asserted "this login has no password" for every user,
|
|
13
|
+
the same falsehood 0.2.2 fixed on the PostgreSQL side. It now reports
|
|
14
|
+
`unknown`, with the reason. A Redshift *group* still reports `no`, because a
|
|
15
|
+
group has no password to hold.
|
|
16
|
+
|
|
17
|
+
- `dp db describe <schema>` now reports `USAGE` grants on Redshift. The query
|
|
18
|
+
read `information_schema.usage_privileges` filtered to `object_type = 'SCHEMA'`,
|
|
19
|
+
which the SQL standard defines over domains, collations and sequences — never
|
|
20
|
+
schemas — so it returned nothing on every server. It now scans
|
|
21
|
+
`has_schema_privilege`, mirroring how the same query has always reported
|
|
22
|
+
`CREATE` on that path. As with `CREATE`, a privilege scan cannot report a
|
|
23
|
+
grantor or a grant option, so both stay empty; the PostgreSQL path reads the
|
|
24
|
+
ACL and does better on both counts.
|
|
25
|
+
|
|
26
|
+
Both fixes rest on documented behaviour and internal precedent rather than a
|
|
27
|
+
live cluster — Redshift cannot be containerized, so CI cannot cover it. See
|
|
28
|
+
below.
|
|
29
|
+
|
|
30
|
+
### Added
|
|
31
|
+
|
|
32
|
+
- A Redshift conformance harness (`tests/integration/redshift/`) for anyone who
|
|
33
|
+
runs dataplat against a real cluster. The read-only tier is safe to point at a
|
|
34
|
+
warehouse in use — a guard refuses anything that is not plainly a read before
|
|
35
|
+
it reaches the server — and it interrogates the assumptions the two fixes above
|
|
36
|
+
depend on, printing what your cluster answered. `CONTRIBUTING.md` documents
|
|
37
|
+
both tiers and the evidence rules for changing SQL that runs on a dialect CI
|
|
38
|
+
cannot reach.
|
|
39
|
+
|
|
40
|
+
## 0.2.2
|
|
41
|
+
|
|
42
|
+
Closes the six defects 0.2.1's integration suite found and pinned as expected
|
|
43
|
+
failures. No expected failures remain.
|
|
44
|
+
|
|
45
|
+
### Fixed
|
|
46
|
+
|
|
47
|
+
- `dp db role show` claimed `Password set: yes` for every role, including a
|
|
48
|
+
passwordless `NOLOGIN` group. The attribute query read `rolpassword` from
|
|
49
|
+
`pg_roles`, whose view definition returns the literal `'********'` and can
|
|
50
|
+
never be NULL. Only `pg_authid` holds the real verifier and it is
|
|
51
|
+
superuser-only, so the field is now tri-state and reports `unknown` — with a
|
|
52
|
+
hint saying why — when the connecting role cannot read it. The privilege is
|
|
53
|
+
probed before the query rather than discovered by catching an error, since a
|
|
54
|
+
permission failure would abort the surrounding transaction.
|
|
55
|
+
|
|
56
|
+
- `dp db describe <schema>` never reported `USAGE` grants. The query read
|
|
57
|
+
`information_schema.usage_privileges`, which on PostgreSQL does not cover
|
|
58
|
+
schemas at all, so the USAGE half of the union always returned nothing. The
|
|
59
|
+
PostgreSQL path now reads the schema ACL directly.
|
|
60
|
+
|
|
61
|
+
- `dp db role show` under-counted a role's tables when it owned a partitioned
|
|
62
|
+
table. Both `relkind` `'r'` and `'p'` map to the label "table" and the
|
|
63
|
+
aggregation assigned rather than accumulated, so one group overwrote the
|
|
64
|
+
other while the total was summed separately and stayed right — the per-schema
|
|
65
|
+
breakdown contradicted its own total.
|
|
66
|
+
|
|
67
|
+
- `dp db dbt-orphans purge` aborted the whole batch when a single relation had
|
|
68
|
+
vanished between scan and purge. The generated statement now uses
|
|
69
|
+
`IF EXISTS`, matching `top-tables`, so a missing relation is a no-op. When a
|
|
70
|
+
dependent object genuinely blocks a drop the purge still stops — that
|
|
71
|
+
all-or-nothing property is deliberate for a destructive batch — but it now
|
|
72
|
+
names the relation and everything depending on it instead of surfacing a raw
|
|
73
|
+
driver error, and records the blockage in the audit log.
|
|
74
|
+
|
|
75
|
+
- `dp db describe <relation>` raised nothing but returned an invalid result for
|
|
76
|
+
a non-view relation: `pg_get_viewdef()` yields NULL rather than erroring, and
|
|
77
|
+
the PostgreSQL branch returned a view definition whose `sql` was `None`,
|
|
78
|
+
violating its own annotation. It now raises, as the Redshift branch already
|
|
79
|
+
did.
|
|
80
|
+
|
|
81
|
+
- `dp db long-queries --history` died with a driver traceback on any server
|
|
82
|
+
where `pg_stat_statements` is installed but not preloaded — the most common
|
|
83
|
+
misconfiguration. The guard that was meant to catch this probed the view's
|
|
84
|
+
columns, which PostgreSQL answers from the view definition without invoking
|
|
85
|
+
the extension, so the probe always succeeded. The failure is now reported as
|
|
86
|
+
an actionable error naming `shared_preload_libraries` and the required
|
|
87
|
+
restart.
|
|
88
|
+
|
|
89
|
+
### Changed
|
|
90
|
+
|
|
91
|
+
- `RoleAttributes.password_set` widened from `bool` to `bool | None`, where
|
|
92
|
+
`None` means "not determinable by this connection". Relevant only if you
|
|
93
|
+
import the dataclass; the CLI renders the third state as `unknown`.
|
|
94
|
+
|
|
95
|
+
- On PostgreSQL, `dp db describe <schema>` privileges now come from the schema
|
|
96
|
+
ACL rather than a `has_schema_privilege` scan. A role holding `CREATE` only
|
|
97
|
+
through membership in a granted role is no longer listed as its own row, and
|
|
98
|
+
`grantor` and `WITH GRANT OPTION` are now real values where the previous
|
|
99
|
+
CREATE half hardcoded them.
|
|
100
|
+
|
|
3
101
|
## 0.2.1
|
|
4
102
|
|
|
5
103
|
### Fixed
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
# Contributing to dataplat
|
|
2
|
+
|
|
3
|
+
## Setup
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
git clone https://github.com/hanslemm/dataplat
|
|
7
|
+
cd dataplat
|
|
8
|
+
uv sync --group dev --all-extras
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## Checks
|
|
12
|
+
|
|
13
|
+
The four gates CI runs, across Python 3.12 and 3.13:
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
uv run pytest
|
|
17
|
+
uv run ruff check .
|
|
18
|
+
uv run ruff format --check .
|
|
19
|
+
uv run mypy dataplat
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
`uv run pytest` is green without Docker: the database-backed tests skip. To run
|
|
23
|
+
them, start a server and point the suite at it:
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
docker run -d --name dp-pg-test \
|
|
27
|
+
-e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=dataplat_test \
|
|
28
|
+
-p 55432:5432 postgres:16 -c shared_preload_libraries=pg_stat_statements
|
|
29
|
+
docker exec dp-pg-test psql -U postgres -d dataplat_test \
|
|
30
|
+
-c 'CREATE EXTENSION IF NOT EXISTS pg_stat_statements'
|
|
31
|
+
|
|
32
|
+
DP_TEST_PG_REQUIRED=1 uv run pytest # everything
|
|
33
|
+
uv run pytest -m "not integration" # skip the database half
|
|
34
|
+
docker rm -f -v dp-pg-test # -v, or the volume dangles
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
`DP_TEST_PG_REQUIRED=1` turns an unreachable server into an error instead of a
|
|
38
|
+
skip. CI sets it; without it a broken database would make the whole suite skip
|
|
39
|
+
and still report success.
|
|
40
|
+
|
|
41
|
+
Commits follow [Conventional Commits](https://www.conventionalcommits.org/).
|
|
42
|
+
|
|
43
|
+
## Testing against a real Redshift cluster
|
|
44
|
+
|
|
45
|
+
Redshift is a managed service, so there is no container and CI cannot cover it.
|
|
46
|
+
If you have a cluster, you can. The suite is in `tests/integration/redshift/` and
|
|
47
|
+
is split into two tiers, because they need different permission to run:
|
|
48
|
+
|
|
49
|
+
| Marker | Mutates? | Needs |
|
|
50
|
+
| --- | --- | --- |
|
|
51
|
+
| `redshift` | no — read-only, safe against a warehouse in use | a reachable cluster |
|
|
52
|
+
| `redshift_ddl` | **yes** | a cluster you can throw away |
|
|
53
|
+
|
|
54
|
+
Credentials come from an ordinary dataplat target, so they stay in your own
|
|
55
|
+
`.envrc` and never reach the repo:
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
export DP_TARGETS=warehouse WAREHOUSE_ENGINE=redshift \
|
|
59
|
+
WAREHOUSE_HOST=... WAREHOUSE_USER=... WAREHOUSE_DATABASE=... \
|
|
60
|
+
WAREHOUSE_PASSWORD=...
|
|
61
|
+
export DP_TEST_RS_TARGET=warehouse # or DP_TEST_RS_DSN=... as an escape hatch
|
|
62
|
+
|
|
63
|
+
DP_TEST_RS_REQUIRED=1 uv run pytest -m redshift # read-only tier
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
| Variable | Effect |
|
|
67
|
+
| --- | --- |
|
|
68
|
+
| `DP_TEST_RS_TARGET` | a dataplat target name, resolved by the tool's own config |
|
|
69
|
+
| `DP_TEST_RS_DSN` | a raw libpq URL, if you would rather not declare a target |
|
|
70
|
+
| `DP_TEST_RS_REQUIRED` | an unreachable cluster becomes an error instead of a skip |
|
|
71
|
+
| `DP_TEST_RS_DISPOSABLE` | **required** before any `redshift_ddl` test will run |
|
|
72
|
+
| `DP_TEST_RS_SCHEMA` | a schema the read-only tier may inspect (otherwise discovered) |
|
|
73
|
+
|
|
74
|
+
A plain `uv run pytest` is unaffected: with nothing configured, both tiers skip.
|
|
75
|
+
|
|
76
|
+
### Why there is a client-side read-only guard
|
|
77
|
+
|
|
78
|
+
`rs_cursor` refuses anything that is not plainly a read *before it is sent*, on
|
|
79
|
+
top of the server-side `READ ONLY` transaction. Two layers, because the cluster
|
|
80
|
+
may be production: Redshift roles are cluster-wide, and its transactional-DDL
|
|
81
|
+
semantics differ from PostgreSQL's, so the rollback-per-test isolation the
|
|
82
|
+
PostgreSQL harness relies on cannot be assumed to clean up a mistake. A
|
|
83
|
+
server-side check would refuse the statement too — but only after it crossed the
|
|
84
|
+
network to a warehouse someone depends on.
|
|
85
|
+
|
|
86
|
+
It denies by default: only `SELECT`, `WITH … SELECT`, `EXPLAIN` and `SHOW` pass.
|
|
87
|
+
It is not fooled by a leading comment, case, a stray semicolon, a second
|
|
88
|
+
statement smuggled after a `SELECT`, a data-modifying CTE, `SELECT … INTO`, or a
|
|
89
|
+
side-effecting builtin such as `pg_terminate_backend`. The statement splitter is
|
|
90
|
+
hand-written rather than regex-based because a regex that ignores quoting can
|
|
91
|
+
*hide* a statement — naive comment stripping turns `SELECT '--' ; DROP TABLE t`
|
|
92
|
+
into a harmless-looking fragment plus a `DROP` the server will happily run. The
|
|
93
|
+
one hole it cannot close is an unlisted side-effecting UDF; that is what the
|
|
94
|
+
server-side layer is for, and `assert_read_only`'s docstring says so.
|
|
95
|
+
|
|
96
|
+
### What a run does and does not prove
|
|
97
|
+
|
|
98
|
+
A green read-only run proves dataplat's `SELECT`s are **valid Redshift SQL
|
|
99
|
+
against a real server, returning results that unpack** — which is precisely the
|
|
100
|
+
class of defect the PostgreSQL suite found repeatedly in these same functions
|
|
101
|
+
(an empty `pg_partition_tree`, a masked column, a view that does not cover
|
|
102
|
+
schemas). It cannot tell you anything about `GRANT`, `DROP`, `RENAME` or session
|
|
103
|
+
termination; those need the DDL tier and a disposable cluster.
|
|
104
|
+
|
|
105
|
+
The run also prints a conformance table of what the cluster answered, because
|
|
106
|
+
the point is learning what the engine does — a green run that recorded nothing
|
|
107
|
+
has taught nobody anything.
|
|
108
|
+
|
|
109
|
+
## Dialect changes: what counts as evidence
|
|
110
|
+
|
|
111
|
+
`dataplat/services/db` targets PostgreSQL and Redshift. PostgreSQL has a real
|
|
112
|
+
integration suite behind it. Redshift has none and cannot get one cheaply — it
|
|
113
|
+
is a managed service, so there is no container to run in CI.
|
|
114
|
+
|
|
115
|
+
For a while the rule was simply "don't touch SQL that runs on Redshift, because
|
|
116
|
+
you can't test it." That is a good instinct and a bad rule. Applied literally it
|
|
117
|
+
blocked seven known defects, and when they were finally looked at one at a time,
|
|
118
|
+
six were fixable and only one genuinely needed Redshift-specific SQL. Five did
|
|
119
|
+
not touch Redshift SQL at all, and the sixth turned out to use a construct the
|
|
120
|
+
codebase was already shipping to Redshift elsewhere.
|
|
121
|
+
|
|
122
|
+
So the question is not "can I test this?" but **"what evidence do I have?"** A
|
|
123
|
+
change affecting the Redshift path needs at least one of the following, in
|
|
124
|
+
descending order of strength:
|
|
125
|
+
|
|
126
|
+
0. **A conformance run confirmed it against a real cluster.** Strongest, and the
|
|
127
|
+
only one that is evidence rather than inference — see the section above. A fix
|
|
128
|
+
currently resting on class 2 should be upgraded to class 0 when someone runs
|
|
129
|
+
the suite, and revisited if the run *refutes* it. `test_conformance.py` names
|
|
130
|
+
the assumptions each shipped fix depends on for exactly this reason.
|
|
131
|
+
|
|
132
|
+
1. **It changes no Redshift SQL.** The fix is pure Python, or touches only a
|
|
133
|
+
`_*_SQL_POSTGRES` constant. Dialect risk is zero by construction — verify
|
|
134
|
+
that claim honestly, then go ahead. Aggregation bugs, error handling, and
|
|
135
|
+
return-value shaping usually land here.
|
|
136
|
+
|
|
137
|
+
2. **The construct is already in production on the Redshift path.** Cite the
|
|
138
|
+
file and line. `ESCAPE '\'` was safe to add to `orphans.py` because
|
|
139
|
+
`top_tables.py` had always sent it to both engines;
|
|
140
|
+
`has_schema_privilege(...)` is safe in `describe.py`'s Redshift branch
|
|
141
|
+
because that branch already calls it. Internal precedent beats
|
|
142
|
+
documentation: it is the same server, the same driver, and code someone is
|
|
143
|
+
already running.
|
|
144
|
+
|
|
145
|
+
3. **It withdraws a claim rather than making one.** Replacing a confidently
|
|
146
|
+
wrong value with "unknown" cannot be more wrong than what it replaced. See
|
|
147
|
+
the standing rule below.
|
|
148
|
+
|
|
149
|
+
4. **Documented Redshift behaviour, cited, plus a fake-cursor test** pinning the
|
|
150
|
+
SQL the Redshift branch emits. Weakest of the four, because documentation and
|
|
151
|
+
deployed reality drift. Use it when the change is worth the residual risk,
|
|
152
|
+
and say so in the commit.
|
|
153
|
+
|
|
154
|
+
If none of the four applies, **do not guess.** Leave the defect, and record it
|
|
155
|
+
in a comment next to the code it affects — not in a tracker nobody reads. The
|
|
156
|
+
comment is what lets the next person re-evaluate instead of rediscovering.
|
|
157
|
+
|
|
158
|
+
### Requirements either way
|
|
159
|
+
|
|
160
|
+
- **Keep the engine constants split.** Never edit a `_*_SQL_REDSHIFT` constant
|
|
161
|
+
to fix a PostgreSQL bug. If a shared statement needs to diverge, split it and
|
|
162
|
+
leave the Redshift half byte-for-byte as it was.
|
|
163
|
+
- **Add a fake-cursor test** asserting what the Redshift branch emits. It is the
|
|
164
|
+
only mechanism that covers that path at all, and it catches the common
|
|
165
|
+
accident of "fixed both branches when I meant one".
|
|
166
|
+
- **Record the evidence in the code**, not just the commit message. A future
|
|
167
|
+
reader deciding whether they may touch the line needs to see why it is the way
|
|
168
|
+
it is.
|
|
169
|
+
|
|
170
|
+
### Standing rule: prefer "unknown" to a confident falsehood
|
|
171
|
+
|
|
172
|
+
`dp db role show` used to print `Password set: yes` for every role, including
|
|
173
|
+
passwordless ones, because the column it read is masked to `'********'` and can
|
|
174
|
+
never be NULL. A report that states something false is worse than one that
|
|
175
|
+
admits a gap — especially a report someone is using for an audit. When the
|
|
176
|
+
server will not tell you, say so, and say why in the same breath: a bare
|
|
177
|
+
"unknown" reads as a tool defect.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dataplat
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.3
|
|
4
4
|
Summary: One command to manage any shape of data platform: databases, ingestion, BI, cloud, and CI.
|
|
5
5
|
Project-URL: Homepage, https://github.com/hanslemm/dataplat
|
|
6
6
|
Project-URL: Repository, https://github.com/hanslemm/dataplat
|
|
@@ -324,6 +324,9 @@ dp ingest airbyte connections set-cursor -c <connection-id> --xmin 0 --yes
|
|
|
324
324
|
|
|
325
325
|
## Development
|
|
326
326
|
|
|
327
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) for the integration suite and the rules
|
|
328
|
+
for changing SQL that runs on Redshift.
|
|
329
|
+
|
|
327
330
|
```bash
|
|
328
331
|
git clone https://github.com/hanslemm/dataplat
|
|
329
332
|
cd dataplat
|
|
@@ -337,6 +340,13 @@ uv run mypy dataplat
|
|
|
337
340
|
CI runs those four across Python 3.12 and 3.13 — the floor the wheel
|
|
338
341
|
advertises as well as the pinned dev version.
|
|
339
342
|
|
|
343
|
+
Redshift cannot be containerized, so CI cannot cover it. If you run dataplat
|
|
344
|
+
against a Redshift cluster, you can verify your own deployment: point
|
|
345
|
+
`DP_TEST_RS_TARGET` at one of your targets and run the read-only tier
|
|
346
|
+
(`uv run pytest -m redshift`). It only issues `SELECT`s — a guard refuses
|
|
347
|
+
anything else before it reaches the server — and prints what your cluster
|
|
348
|
+
answered. See [CONTRIBUTING.md](CONTRIBUTING.md#testing-against-a-real-redshift-cluster).
|
|
349
|
+
|
|
340
350
|
### Integration tests against a real PostgreSQL
|
|
341
351
|
|
|
342
352
|
Most of the suite drives a fake database cursor. That proves a code path
|
|
@@ -279,6 +279,9 @@ dp ingest airbyte connections set-cursor -c <connection-id> --xmin 0 --yes
|
|
|
279
279
|
|
|
280
280
|
## Development
|
|
281
281
|
|
|
282
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) for the integration suite and the rules
|
|
283
|
+
for changing SQL that runs on Redshift.
|
|
284
|
+
|
|
282
285
|
```bash
|
|
283
286
|
git clone https://github.com/hanslemm/dataplat
|
|
284
287
|
cd dataplat
|
|
@@ -292,6 +295,13 @@ uv run mypy dataplat
|
|
|
292
295
|
CI runs those four across Python 3.12 and 3.13 — the floor the wheel
|
|
293
296
|
advertises as well as the pinned dev version.
|
|
294
297
|
|
|
298
|
+
Redshift cannot be containerized, so CI cannot cover it. If you run dataplat
|
|
299
|
+
against a Redshift cluster, you can verify your own deployment: point
|
|
300
|
+
`DP_TEST_RS_TARGET` at one of your targets and run the read-only tier
|
|
301
|
+
(`uv run pytest -m redshift`). It only issues `SELECT`s — a guard refuses
|
|
302
|
+
anything else before it reaches the server — and prints what your cluster
|
|
303
|
+
answered. See [CONTRIBUTING.md](CONTRIBUTING.md#testing-against-a-real-redshift-cluster).
|
|
304
|
+
|
|
295
305
|
### Integration tests against a real PostgreSQL
|
|
296
306
|
|
|
297
307
|
Most of the suite drives a fake database cursor. That proves a code path
|
|
@@ -20,6 +20,8 @@ from dataplat.services.db.connection import SqlEngine
|
|
|
20
20
|
from dataplat.services.db.orphans import (
|
|
21
21
|
DEPRECATED_SUFFIX,
|
|
22
22
|
LIVE_STATUSES,
|
|
23
|
+
BlockedEntry,
|
|
24
|
+
DependentObjectsError,
|
|
23
25
|
DropEntry,
|
|
24
26
|
ObjectKind,
|
|
25
27
|
RenameEntry,
|
|
@@ -661,7 +663,19 @@ def purge_cmd(
|
|
|
661
663
|
help="Path to a file with one exclusion token per line.",
|
|
662
664
|
),
|
|
663
665
|
) -> None:
|
|
664
|
-
"""Permanently drop every object ending in _deprecated (irreversible).
|
|
666
|
+
"""Permanently drop every object ending in _deprecated (irreversible).
|
|
667
|
+
|
|
668
|
+
All-or-nothing per target, by design: every drop for one warehouse runs
|
|
669
|
+
inside a single transaction, so if the warehouse refuses one of them
|
|
670
|
+
nothing is dropped for that target and the error names the blocking
|
|
671
|
+
relation. Keeping the drops that already succeeded would need either a
|
|
672
|
+
SAVEPOINT per object (Redshift has none) or a commit per object (which
|
|
673
|
+
would break the guarantee that --dry-run writes nothing) — and for a
|
|
674
|
+
destructive batch, stopping on the first surprise is the safer default. No
|
|
675
|
+
CASCADE is issued either, so a live view or foreign key still pointing at a
|
|
676
|
+
_deprecated relation is a real signal: resolve the dependents the error
|
|
677
|
+
lists, then re-run.
|
|
678
|
+
"""
|
|
665
679
|
if log is None:
|
|
666
680
|
log = _timestamped_log_path(PURGE_LOG_PREFIX)
|
|
667
681
|
|
|
@@ -691,24 +705,48 @@ def purge_cmd(
|
|
|
691
705
|
)
|
|
692
706
|
|
|
693
707
|
all_drops: list[DropEntry] = []
|
|
708
|
+
blocked: list[BlockedEntry] = []
|
|
694
709
|
try:
|
|
695
710
|
for label, engine, env_prefix in engines:
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
711
|
+
try:
|
|
712
|
+
all_drops.extend(
|
|
713
|
+
_purge_for_engine(
|
|
714
|
+
label,
|
|
715
|
+
engine,
|
|
716
|
+
env_prefix=env_prefix,
|
|
717
|
+
excluded_user_schemas=excluded_user_schemas,
|
|
718
|
+
excluded_user_relations=excluded_user_relations,
|
|
719
|
+
dry_run=dry_run,
|
|
720
|
+
renamed_at=renamed_at,
|
|
721
|
+
cutoff=cutoff,
|
|
722
|
+
include_unknown=include_unknown,
|
|
723
|
+
)
|
|
707
724
|
)
|
|
708
|
-
|
|
725
|
+
except DependentObjectsError as exc:
|
|
726
|
+
# The engine's transaction is already rolled back, so the drops
|
|
727
|
+
# it had made are (correctly) absent from all_drops. Record the
|
|
728
|
+
# refused attempt so the audit log shows why this target did
|
|
729
|
+
# nothing, then re-raise with the [<engine>] prefix every other
|
|
730
|
+
# error from this command carries.
|
|
731
|
+
blocked.append(
|
|
732
|
+
BlockedEntry(
|
|
733
|
+
database=label,
|
|
734
|
+
schema=exc.schema,
|
|
735
|
+
name=exc.name,
|
|
736
|
+
kind=exc.kind,
|
|
737
|
+
dependents=exc.dependents,
|
|
738
|
+
)
|
|
739
|
+
)
|
|
740
|
+
raise ServiceError(f"[{label}] {exc}") from exc
|
|
709
741
|
except ServiceError as exc:
|
|
710
|
-
_write_purge_log(log, all_drops, dry_run=dry_run)
|
|
742
|
+
_write_purge_log(log, all_drops, dry_run=dry_run, blocked=blocked)
|
|
711
743
|
console.print(f"[red]{esc(exc)}[/red]")
|
|
744
|
+
if blocked:
|
|
745
|
+
console.print(
|
|
746
|
+
"[yellow]Nothing was dropped for that target: the purge is one "
|
|
747
|
+
"transaction, so a refused drop rolls the whole batch "
|
|
748
|
+
"back.[/yellow]"
|
|
749
|
+
)
|
|
712
750
|
console.print(
|
|
713
751
|
f"[yellow]Partial purge log written to {esc(log)} "
|
|
714
752
|
f"({len(all_drops)} entries).[/yellow]"
|
|
@@ -839,12 +877,22 @@ def _drop_one(
|
|
|
839
877
|
return DropEntry(database=label, schema=schema, name=name, kind=kind)
|
|
840
878
|
|
|
841
879
|
|
|
842
|
-
def _write_purge_log(
|
|
880
|
+
def _write_purge_log(
|
|
881
|
+
log_path: str,
|
|
882
|
+
entries: list[DropEntry],
|
|
883
|
+
*,
|
|
884
|
+
dry_run: bool,
|
|
885
|
+
blocked: list[BlockedEntry] | None = None,
|
|
886
|
+
) -> None:
|
|
843
887
|
payload = {
|
|
844
888
|
"generated_at": datetime.now(UTC).isoformat(),
|
|
845
889
|
"dry_run": dry_run,
|
|
846
890
|
"source": "dbt-orphans-purge",
|
|
847
891
|
"drops": entries,
|
|
892
|
+
# A refused drop rolls its target's transaction back, so "drops" cannot
|
|
893
|
+
# hold the attempt; recorded separately or the log would show a purge
|
|
894
|
+
# that raised as having done nothing at all.
|
|
895
|
+
"blocked": blocked or [],
|
|
848
896
|
}
|
|
849
897
|
with open(log_path, "w") as f:
|
|
850
898
|
json.dump(payload, f, indent=4)
|
|
@@ -72,6 +72,26 @@ def _more_line(hidden: int) -> str:
|
|
|
72
72
|
return f" [dim italic]… and {hidden} more (raise --limit to see all).[/dim italic]"
|
|
73
73
|
|
|
74
74
|
|
|
75
|
+
def _password_set_value(password_set: bool | None, engine: SqlEngine) -> str:
|
|
76
|
+
"""Render the tri-state ``password_set`` as markup.
|
|
77
|
+
|
|
78
|
+
``None`` means the server would not say, and the reason travels with the
|
|
79
|
+
word: a bare "unknown" in a security report reads as a tool defect, and
|
|
80
|
+
rendering it as "no" would be a false negative on exactly the field an
|
|
81
|
+
auditor came for. The reason differs by engine, and naming pg_authid on
|
|
82
|
+
Redshift — which has no such relation — would send the reader somewhere
|
|
83
|
+
that does not exist.
|
|
84
|
+
"""
|
|
85
|
+
if password_set is None:
|
|
86
|
+
reason = (
|
|
87
|
+
"no readable password catalog on Redshift"
|
|
88
|
+
if engine == SqlEngine.redshift
|
|
89
|
+
else "needs superuser to read pg_authid"
|
|
90
|
+
)
|
|
91
|
+
return f"unknown [dim]({reason})[/dim]"
|
|
92
|
+
return "yes" if password_set else "no"
|
|
93
|
+
|
|
94
|
+
|
|
75
95
|
def _attributes_metadata(attrs: RoleAttributes) -> list[tuple[str, str]]:
|
|
76
96
|
flags: list[str] = []
|
|
77
97
|
if attrs.superuser:
|
|
@@ -97,13 +117,20 @@ def _attributes_metadata(attrs: RoleAttributes) -> list[tuple[str, str]]:
|
|
|
97
117
|
# The title card renders metadata values as markup, and this one is
|
|
98
118
|
# warehouse data (rolvaliduntil rendered by the driver).
|
|
99
119
|
metadata.append(("Valid until", esc(attrs.valid_until)))
|
|
100
|
-
|
|
120
|
+
# `is True` on purpose: password_set is tri-state, and the card has room
|
|
121
|
+
# only for a bare word. An unexplained "unknown" chip is worse than no
|
|
122
|
+
# chip, so the unknown case is left to the Attributes table, which has
|
|
123
|
+
# room to say why.
|
|
124
|
+
if attrs.password_set is True:
|
|
101
125
|
metadata.append(("Password", "set"))
|
|
102
126
|
return metadata
|
|
103
127
|
|
|
104
128
|
|
|
105
129
|
def _render_attributes(
|
|
106
|
-
console: Console,
|
|
130
|
+
console: Console,
|
|
131
|
+
counter: _SectionCounter,
|
|
132
|
+
attrs: RoleAttributes,
|
|
133
|
+
engine: SqlEngine,
|
|
107
134
|
) -> None:
|
|
108
135
|
_print_section_heading(
|
|
109
136
|
console, counter, "Attributes", "Role flags, login, and limits."
|
|
@@ -122,7 +149,7 @@ def _render_attributes(
|
|
|
122
149
|
"Connection limit",
|
|
123
150
|
"unlimited" if attrs.connection_limit < 0 else str(attrs.connection_limit),
|
|
124
151
|
)
|
|
125
|
-
table.add_row("Password set",
|
|
152
|
+
table.add_row("Password set", _password_set_value(attrs.password_set, engine))
|
|
126
153
|
table.add_row("Valid until", cell(attrs.valid_until or "—"))
|
|
127
154
|
console.print(_indent(table))
|
|
128
155
|
|
|
@@ -328,7 +355,7 @@ def render_role_description(
|
|
|
328
355
|
console.print()
|
|
329
356
|
|
|
330
357
|
counter = _SectionCounter()
|
|
331
|
-
_render_attributes(console, counter, desc.attributes)
|
|
358
|
+
_render_attributes(console, counter, desc.attributes, engine)
|
|
332
359
|
parent_caption = (
|
|
333
360
|
"Roles this role is a member of (single-level; Redshift groups)."
|
|
334
361
|
if engine == SqlEngine.redshift
|
|
@@ -665,6 +665,12 @@ def fetch_view_definition(cursor: Any, oid: int, engine: SqlEngine) -> ViewDefin
|
|
|
665
665
|
if row is None:
|
|
666
666
|
raise TargetNotFoundError(f"view with oid {oid} not found")
|
|
667
667
|
sql, is_updatable, check_option = row
|
|
668
|
+
# pg_get_viewdef() does not error on a non-view oid, it returns NULL, so a
|
|
669
|
+
# table's oid produces a row whose definition is missing. Reject it the way
|
|
670
|
+
# the Redshift branch above does: returning ViewDefinition(sql=None) would
|
|
671
|
+
# break this dataclass's own `sql: str` contract for every caller.
|
|
672
|
+
if sql is None:
|
|
673
|
+
raise TargetNotFoundError(f"view with oid {oid} not found")
|
|
668
674
|
updatable = isinstance(is_updatable, str) and is_updatable.upper() == "YES"
|
|
669
675
|
if check_option is None or check_option == "NONE":
|
|
670
676
|
check_option_value: str | None = None
|
|
@@ -752,10 +758,58 @@ ORDER BY c.relkind, c.relname
|
|
|
752
758
|
"""
|
|
753
759
|
|
|
754
760
|
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
761
|
+
# PostgreSQL exposes schema ACLs only through pg_namespace.nspacl; there is no
|
|
762
|
+
# information_schema view for them. usage_privileges reports DOMAIN, COLLATION,
|
|
763
|
+
# FDW, foreign server and sequence object types and never 'SCHEMA', so the
|
|
764
|
+
# USAGE half of the old shared query silently matched zero rows and every
|
|
765
|
+
# GRANT USAGE ON SCHEMA was missing from the report.
|
|
766
|
+
#
|
|
767
|
+
# The COALESCE is load-bearing: nspacl stays NULL until the first GRANT or
|
|
768
|
+
# REVOKE touches the schema, and NULL means "the built-in default" -- the owner
|
|
769
|
+
# holding USAGE + CREATE -- not "nobody holds anything". acldefault('n', owner)
|
|
770
|
+
# materialises that default so a freshly created schema still reports its
|
|
771
|
+
# owner's privileges.
|
|
772
|
+
#
|
|
773
|
+
# Reading the ACL also makes grantor and WITH GRANT OPTION truthful: the CREATE
|
|
774
|
+
# half of the old query derived grantees from has_schema_privilege() and had to
|
|
775
|
+
# hardcode grantor='' and with_grant_option=false.
|
|
776
|
+
_SCHEMA_PRIVILEGES_SQL_POSTGRES = """
|
|
777
|
+
SELECT
|
|
778
|
+
CASE WHEN (acl).grantee = 0 THEN 'PUBLIC'
|
|
779
|
+
ELSE pg_get_userbyid((acl).grantee) END AS grantee,
|
|
780
|
+
(acl).privilege_type AS privilege_type,
|
|
781
|
+
(acl).is_grantable AS with_grant_option,
|
|
782
|
+
pg_get_userbyid((acl).grantor) AS grantor
|
|
783
|
+
FROM pg_namespace n
|
|
784
|
+
CROSS JOIN LATERAL aclexplode(COALESCE(n.nspacl, acldefault('n', n.nspowner))) AS acl
|
|
785
|
+
WHERE n.nspname = %s
|
|
786
|
+
ORDER BY grantee, privilege_type
|
|
787
|
+
"""
|
|
788
|
+
|
|
789
|
+
|
|
790
|
+
# Not the aclexplode form above: Redshift has no aclexplode(), and no cluster is
|
|
791
|
+
# available to test a replacement against.
|
|
792
|
+
#
|
|
793
|
+
# The USAGE half used to read information_schema.usage_privileges filtered to
|
|
794
|
+
# object_type = 'SCHEMA', which the SQL standard defines over domains,
|
|
795
|
+
# collations and sequences — never schemas. It returned zero rows on every
|
|
796
|
+
# server, so a GRANT USAGE ON SCHEMA was invisible here exactly as it was on
|
|
797
|
+
# PostgreSQL. It is replaced by a has_schema_privilege() scan mirroring the
|
|
798
|
+
# CREATE half directly below, which this same query has always run against
|
|
799
|
+
# Redshift: same function, same catalog, same shape, only the privilege string
|
|
800
|
+
# differs (CONTRIBUTING, evidence 2 — internal precedent on this very path).
|
|
801
|
+
#
|
|
802
|
+
# Known limitation, shared with the CREATE half and unchanged: a privilege scan
|
|
803
|
+
# cannot report a grantor or a grant option, so both are reported empty, and
|
|
804
|
+
# roles holding the privilege only through membership appear as their own rows.
|
|
805
|
+
# The PostgreSQL path reads the ACL and does better on both counts.
|
|
806
|
+
_SCHEMA_PRIVILEGES_SQL_REDSHIFT = """
|
|
807
|
+
SELECT grantee, 'USAGE', false, ''
|
|
808
|
+
FROM (
|
|
809
|
+
SELECT r.rolname AS grantee
|
|
810
|
+
FROM pg_roles r
|
|
811
|
+
WHERE has_schema_privilege(r.rolname, %s, 'USAGE')
|
|
812
|
+
) u
|
|
759
813
|
UNION ALL
|
|
760
814
|
SELECT grantee, 'CREATE', false, ''
|
|
761
815
|
FROM (
|
|
@@ -820,9 +874,19 @@ def fetch_schema_contents(
|
|
|
820
874
|
]
|
|
821
875
|
|
|
822
876
|
|
|
823
|
-
def fetch_schema_privileges(
|
|
824
|
-
|
|
825
|
-
|
|
877
|
+
def fetch_schema_privileges(
|
|
878
|
+
cursor: Any, schema: str, engine: SqlEngine
|
|
879
|
+
) -> list[PrivilegeGrant]:
|
|
880
|
+
"""Return USAGE + CREATE grants for the schema.
|
|
881
|
+
|
|
882
|
+
On PostgreSQL these come from ``pg_namespace.nspacl``, which lists explicit
|
|
883
|
+
ACL entries plus the owner's implicit default; a role that only holds the
|
|
884
|
+
privilege through membership in a granted role is not a separate row.
|
|
885
|
+
"""
|
|
886
|
+
if engine == SqlEngine.redshift:
|
|
887
|
+
cursor.execute(_SCHEMA_PRIVILEGES_SQL_REDSHIFT, (schema, schema))
|
|
888
|
+
else:
|
|
889
|
+
cursor.execute(_SCHEMA_PRIVILEGES_SQL_POSTGRES, (schema,))
|
|
826
890
|
return [PrivilegeGrant(*row) for row in cursor.fetchall()]
|
|
827
891
|
|
|
828
892
|
|
|
@@ -1044,7 +1108,7 @@ def describe_schema(
|
|
|
1044
1108
|
) -> SchemaDescription:
|
|
1045
1109
|
"""Compose a full schema description by invoking each fetcher."""
|
|
1046
1110
|
header = fetch_schema_header(cursor, ref.schema)
|
|
1047
|
-
privileges = fetch_schema_privileges(cursor, ref.schema)
|
|
1111
|
+
privileges = fetch_schema_privileges(cursor, ref.schema, engine)
|
|
1048
1112
|
default_privileges = fetch_schema_default_privileges(cursor, ref.schema, engine)
|
|
1049
1113
|
contents = fetch_schema_contents(cursor, ref.schema, engine)
|
|
1050
1114
|
return SchemaDescription(
|