truequery 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- truequery/__init__.py +3 -0
- truequery/analyzers/__init__.py +0 -0
- truequery/analyzers/project_checks.py +220 -0
- truequery/analyzers/sql.py +529 -0
- truequery/checks/__init__.py +0 -0
- truequery/checks/sql.py +1278 -0
- truequery/cli.py +710 -0
- truequery/engine.py +346 -0
- truequery/hook.py +860 -0
- truequery/keylookup.py +135 -0
- truequery/models.py +32 -0
- truequery/project.py +561 -0
- truequery/registry_impl.json +127 -0
- truequery/report.py +343 -0
- truequery/schema.py +206 -0
- truequery/settings.py +239 -0
- truequery/standard/consume/README.md +111 -0
- truequery/standard/consume/adhoc-limit-required.yaml +50 -0
- truequery/standard/consume/aggregate-before-export.yaml +39 -0
- truequery/standard/consume/consumer-cost-guard.yaml +44 -0
- truequery/standard/consume/consumer-reads-marts.yaml +38 -0
- truequery/standard/consume/dashboard-query-budget.yaml +40 -0
- truequery/standard/consume/dashboard-refresh-schedule.yaml +47 -0
- truequery/standard/consume/deterministic-export-order.yaml +44 -0
- truequery/standard/consume/explicit-timezone-consumers.yaml +44 -0
- truequery/standard/consume/explore-answer-carries-reproducible-query.yaml +56 -0
- truequery/standard/consume/explore-caveats-and-unchecked-stated.yaml +52 -0
- truequery/standard/consume/explore-day-boundaries-and-partial-periods.yaml +51 -0
- truequery/standard/consume/explore-experiment-readout-follows-plan.yaml +63 -0
- truequery/standard/consume/explore-freshness-and-coverage-checked.yaml +54 -0
- truequery/standard/consume/explore-inspect-distributions-and-examples.yaml +47 -0
- truequery/standard/consume/explore-join-row-count-checks.yaml +53 -0
- truequery/standard/consume/explore-null-aware-negative-filters.yaml +50 -0
- truequery/standard/consume/explore-question-metric-grain-stated.yaml +55 -0
- truequery/standard/consume/explore-rates-not-counts-across-segments.yaml +51 -0
- truequery/standard/consume/explore-reconcile-to-known-total.yaml +51 -0
- truequery/standard/consume/explore-reuse-governed-metric.yaml +52 -0
- truequery/standard/consume/export-row-cap.yaml +45 -0
- truequery/standard/consume/freshness-displayed.yaml +44 -0
- truequery/standard/consume/no-deep-offset.yaml +38 -0
- truequery/standard/consume/parameterized-dashboard-filters.yaml +38 -0
- truequery/standard/consume/reverse-etl-idempotent-keys.yaml +45 -0
- truequery/standard/consume/reverse-etl-pii-consent.yaml +50 -0
- truequery/standard/consume/reverse-etl-schema-contract.yaml +47 -0
- truequery/standard/dq/README.md +117 -0
- truequery/standard/dq/accuracy-proxy-or-unmeasurable.yaml +45 -0
- truequery/standard/dq/audits-blocking-every-run.yaml +47 -0
- truequery/standard/dq/checks-versioned-with-code.yaml +52 -0
- truequery/standard/dq/completeness-declare-missing-sentinels.yaml +44 -0
- truequery/standard/dq/completeness-fill-rate-thresholds.yaml +45 -0
- truequery/standard/dq/completeness-not-null-required-columns.yaml +60 -0
- truequery/standard/dq/completeness-table-not-empty.yaml +48 -0
- truequery/standard/dq/consistency-cross-column-row-rules.yaml +49 -0
- truequery/standard/dq/consistency-no-orphan-keys.yaml +49 -0
- truequery/standard/dq/consistency-paired-aggregate-reconciliation.yaml +48 -0
- truequery/standard/dq/schema-conformance-checked-on-read.yaml +48 -0
- truequery/standard/dq/unenforced-constraints-are-tested.yaml +51 -0
- truequery/standard/dq/uniqueness-composite-grain.yaml +52 -0
- truequery/standard/dq/uniqueness-key-no-duplicates.yaml +53 -0
- truequery/standard/dq/unit-tests-with-fixtures.yaml +52 -0
- truequery/standard/dq/validity-allowed-value-sets.yaml +46 -0
- truequery/standard/dq/validity-enforce-model-contracts.yaml +51 -0
- truequery/standard/dq/validity-range-format-type-rules.yaml +50 -0
- truequery/standard/gov/README.md +94 -0
- truequery/standard/gov/audit-columns-or-snapshots.yaml +45 -0
- truequery/standard/gov/column-grants-for-pii.yaml +41 -0
- truequery/standard/gov/declared-model-owner.yaml +43 -0
- truequery/standard/gov/description-coverage-for-shared-models.yaml +40 -0
- truequery/standard/gov/erasure-path-documented.yaml +43 -0
- truequery/standard/gov/exposures-registered.yaml +73 -0
- truequery/standard/gov/group-private-access.yaml +41 -0
- truequery/standard/gov/least-privilege-grants.yaml +80 -0
- truequery/standard/gov/lineage-from-declared-sources.yaml +41 -0
- truequery/standard/gov/mesh-public-contracts-only.yaml +41 -0
- truequery/standard/gov/no-bypassrls-roles.yaml +39 -0
- truequery/standard/gov/no-public-privileges.yaml +39 -0
- truequery/standard/gov/no-raw-pii-in-seeds.yaml +41 -0
- truequery/standard/gov/pii-classification-declared.yaml +42 -0
- truequery/standard/gov/production-pii-not-in-dev.yaml +43 -0
- truequery/standard/gov/redacting-views-for-pii.yaml +41 -0
- truequery/standard/gov/retention-declared.yaml +42 -0
- truequery/standard/gov/row-policy-for-sensitive-tables.yaml +38 -0
- truequery/standard/ingest/README.md +100 -0
- truequery/standard/ingest/cap-intake-cost-and-volume.yaml +46 -0
- truequery/standard/ingest/cdc-replica-identity-required.yaml +47 -0
- truequery/standard/ingest/checkpoint-streaming-intake.yaml +42 -0
- truequery/standard/ingest/choose-batch-vs-streaming-deliberately.yaml +45 -0
- truequery/standard/ingest/declare-source-freshness.yaml +80 -0
- truequery/standard/ingest/dedupe-on-natural-key-before-append.yaml +45 -0
- truequery/standard/ingest/handle-schema-drift-explicitly.yaml +45 -0
- truequery/standard/ingest/idempotent-batch-loads.yaml +49 -0
- truequery/standard/ingest/isolate-source-failures.yaml +50 -0
- truequery/standard/ingest/landing-rebuildable-from-source.yaml +43 -0
- truequery/standard/ingest/monitor-cdc-lag-and-slots.yaml +47 -0
- truequery/standard/ingest/pin-connector-and-schema-versions.yaml +42 -0
- truequery/standard/ingest/pin-file-format-options.yaml +42 -0
- truequery/standard/ingest/propagate-deletes-explicitly.yaml +48 -0
- truequery/standard/ingest/quarantine-invalid-rows-at-intake.yaml +56 -0
- truequery/standard/ingest/raw-landing-append-only.yaml +48 -0
- truequery/standard/ingest/reconcile-row-counts-per-load.yaml +43 -0
- truequery/standard/ingest/record-file-lineage-at-intake.yaml +48 -0
- truequery/standard/ingest/reference-sources-not-raw-tables.yaml +42 -0
- truequery/standard/ingest/require-completeness-marker.yaml +50 -0
- truequery/standard/ingest/scan-pii-at-landing.yaml +45 -0
- truequery/standard/ingest/seeds-small-static-only.yaml +43 -0
- truequery/standard/ingest/test-intake-against-fixtures.yaml +46 -0
- truequery/standard/ingest/watermark-and-late-data-policy.yaml +49 -0
- truequery/standard/model/README.md +91 -0
- truequery/standard/model/dbt-contract-enforced.yaml +50 -0
- truequery/standard/model/delete-handling-incremental.yaml +51 -0
- truequery/standard/model/explicit-column-lists.yaml +83 -0
- truequery/standard/model/full-refresh-parity.yaml +49 -0
- truequery/standard/model/grain-preserved-joins.yaml +52 -0
- truequery/standard/model/idempotent-reexec.yaml +48 -0
- truequery/standard/model/incremental-lookback-window.yaml +53 -0
- truequery/standard/model/incremental-watermark-filter.yaml +48 -0
- truequery/standard/model/jinja-macro-determinism.yaml +46 -0
- truequery/standard/model/jinja-no-nested-curlies.yaml +43 -0
- truequery/standard/model/materialization-choice.yaml +51 -0
- truequery/standard/model/merge-source-deduped.yaml +57 -0
- truequery/standard/model/microbatch-event-time.yaml +39 -0
- truequery/standard/model/null-join-keys-excluded.yaml +45 -0
- truequery/standard/model/snapshot-scd2-validity.yaml +45 -0
- truequery/standard/model/snapshot-strategy-choice.yaml +41 -0
- truequery/standard/model/staging-no-business-logic.yaml +44 -0
- truequery/standard/model/surrogate-keys-stable.yaml +37 -0
- truequery/standard/model/unique-key-merge-key.yaml +55 -0
- truequery/standard/ops/README.md +98 -0
- truequery/standard/ops/breaking-change-categorization.yaml +35 -0
- truequery/standard/ops/ci-pr-checks-required.yaml +44 -0
- truequery/standard/ops/ci-slim-builds-only-changed.yaml +42 -0
- truequery/standard/ops/cost-estimate-before-run.yaml +41 -0
- truequery/standard/ops/cost-guardrail-confidence-levels.yaml +39 -0
- truequery/standard/ops/dev-environment-expiry.yaml +42 -0
- truequery/standard/ops/disposable-dev-environments.yaml +42 -0
- truequery/standard/ops/drift-detection-history-first.yaml +42 -0
- truequery/standard/ops/full-refresh-approval.yaml +76 -0
- truequery/standard/ops/model-versioning-breaking-changes.yaml +83 -0
- truequery/standard/ops/plan-preview-before-apply.yaml +39 -0
- truequery/standard/ops/post-deploy-data-validation.yaml +42 -0
- truequery/standard/ops/pre-destructive-snapshot.yaml +35 -0
- truequery/standard/ops/prod-credentials-isolation.yaml +96 -0
- truequery/standard/ops/run-artifact-retention.yaml +41 -0
- truequery/standard/ops/staged-upgrades-with-backup.yaml +39 -0
- truequery/standard/orch/README.md +74 -0
- truequery/standard/orch/concurrency-pools-and-limits.yaml +63 -0
- truequery/standard/orch/dag-ci-parse-and-unit-tests.yaml +70 -0
- truequery/standard/orch/data-aware-scheduling-over-polling.yaml +62 -0
- truequery/standard/orch/declared-asset-lineage.yaml +66 -0
- truequery/standard/orch/deferrable-sensors-with-timeout.yaml +75 -0
- truequery/standard/orch/explicit-catchup-policy.yaml +53 -0
- truequery/standard/orch/explicit-schedule-timezone.yaml +68 -0
- truequery/standard/orch/failure-alert-routing.yaml +100 -0
- truequery/standard/orch/idempotent-backfill-windowing.yaml +68 -0
- truequery/standard/orch/idempotent-retries-with-limits.yaml +79 -0
- truequery/standard/orch/light-dag-parse-no-heavy-top-level.yaml +75 -0
- truequery/standard/orch/no-circular-dependencies.yaml +67 -0
- truequery/standard/orch/partitioned-assets-windowing.yaml +66 -0
- truequery/standard/orch/sla-deadline-alerts.yaml +76 -0
- truequery/standard/orch/task-atomicity-no-partial-writes.yaml +62 -0
- truequery/standard/orch/task-timeout-execution-limit.yaml +70 -0
- truequery/standard/py/README.md +63 -0
- truequery/standard/py/airflow-isolate-task-dependencies.yaml +40 -0
- truequery/standard/py/dagster-asset-declared-dependencies.yaml +47 -0
- truequery/standard/py/dagster-io-manager-not-manual-io.yaml +47 -0
- truequery/standard/py/dagster-resources-for-external-clients.yaml +53 -0
- truequery/standard/py/dbt-python-model-contract.yaml +48 -0
- truequery/standard/py/deterministic-model-outputs.yaml +83 -0
- truequery/standard/py/hook-adapter-connection-lifecycle.yaml +59 -0
- truequery/standard/py/idempotent-task-logic.yaml +55 -0
- truequery/standard/py/no-n-plus-one-queries-in-loops.yaml +46 -0
- truequery/standard/py/pandas-filter-before-collect.yaml +44 -0
- truequery/standard/py/pandas-no-chained-assignment.yaml +41 -0
- truequery/standard/py/pandas-no-row-loop-mutations.yaml +44 -0
- truequery/standard/py/pin-pipeline-dependencies.yaml +82 -0
- truequery/standard/py/sqlmesh-audit-python-outputs.yaml +49 -0
- truequery/standard/py/sqlmesh-python-model-execution-context.yaml +52 -0
- truequery/standard/sem/README.md +97 -0
- truequery/standard/sem/aggregation-type-matches-additivity.yaml +61 -0
- truequery/standard/sem/conversion-metrics-window-bounds.yaml +54 -0
- truequery/standard/sem/count-distinct-uses-entity-key.yaml +52 -0
- truequery/standard/sem/cumulative-metrics-time-spine.yaml +53 -0
- truequery/standard/sem/derived-metrics-compose-from-governed.yaml +52 -0
- truequery/standard/sem/description-and-ownership-per-metric.yaml +50 -0
- truequery/standard/sem/dimension-naming-conventions.yaml +52 -0
- truequery/standard/sem/entity-primary-key-declared.yaml +54 -0
- truequery/standard/sem/join-paths-declared-via-entities.yaml +47 -0
- truequery/standard/sem/metric-filters-declared-not-hidden.yaml +57 -0
- truequery/standard/sem/metric-grain-declaration.yaml +52 -0
- truequery/standard/sem/metric-smoke-test-on-change.yaml +43 -0
- truequery/standard/sem/no-fanout-double-counting.yaml +52 -0
- truequery/standard/sem/no-metric-defined-off-staging.yaml +47 -0
- truequery/standard/sem/null-handling-in-aggregations.yaml +57 -0
- truequery/standard/sem/percentile-metrics-use-approximate.yaml +51 -0
- truequery/standard/sem/ratio-metrics-divide-after-aggregation.yaml +49 -0
- truequery/standard/sem/saved-query-canonical-slices.yaml +56 -0
- truequery/standard/sem/semantic-model-validates.yaml +50 -0
- truequery/standard/sem/single-governed-metric-definition.yaml +68 -0
- truequery/standard/sem/time-dimension-standardized.yaml +51 -0
- truequery/standard/sem/unit-consistency.yaml +54 -0
- truequery/standard/sql/README.md +126 -0
- truequery/standard/sql/accidental-cross-join.yaml +58 -0
- truequery/standard/sql/average-of-ratios.yaml +63 -0
- truequery/standard/sql/between-date-on-timestamp.yaml +74 -0
- truequery/standard/sql/case-without-else-in-aggregate.yaml +67 -0
- truequery/standard/sql/count-star-after-left-join.yaml +67 -0
- truequery/standard/sql/distinct-dedup-bandaid.yaml +59 -0
- truequery/standard/sql/explore-limit-is-not-a-sample.yaml +56 -0
- truequery/standard/sql/explore-profile-filter-columns.yaml +48 -0
- truequery/standard/sql/fanout-join-pre-aggregation.yaml +64 -0
- truequery/standard/sql/group-by-bare-column.yaml +75 -0
- truequery/standard/sql/insert-explicit-column-list.yaml +60 -0
- truequery/standard/sql/integer-division-truncation.yaml +94 -0
- truequery/standard/sql/keyset-pagination.yaml +70 -0
- truequery/standard/sql/left-join-null-rejecting-where.yaml +58 -0
- truequery/standard/sql/limit-requires-order-by.yaml +67 -0
- truequery/standard/sql/natural-join-explicit-keys.yaml +56 -0
- truequery/standard/sql/negative-filter-drops-nulls.yaml +70 -0
- truequery/standard/sql/nondeterministic-merge-keys.yaml +69 -0
- truequery/standard/sql/not-in-nullable-subquery.yaml +57 -0
- truequery/standard/sql/null-equality-comparison.yaml +65 -0
- truequery/standard/sql/random-sampling-no-sort.yaml +48 -0
- truequery/standard/sql/row-number-dedupe-tiebreaker.yaml +67 -0
- truequery/standard/sql/sargable-filter-bare-column.yaml +53 -0
- truequery/standard/sql/set-operation-column-position.yaml +59 -0
- truequery/standard/sql/unguarded-update-delete.yaml +63 -0
- truequery/standard/sql/union-all-default.yaml +70 -0
- truequery/standard/store/README.md +88 -0
- truequery/standard/store/cluster-on-frequent-filters-joins.yaml +44 -0
- truequery/standard/store/compact-small-files-optimize.yaml +44 -0
- truequery/standard/store/declare-not-null-check-constraints.yaml +43 -0
- truequery/standard/store/declare-sort-order-for-range-queries.yaml +44 -0
- truequery/standard/store/enable-change-feed-for-incremental-consumers.yaml +44 -0
- truequery/standard/store/evolve-partitions-by-metadata-not-rewrite.yaml +45 -0
- truequery/standard/store/explicit-physical-types-no-implicit-casts.yaml +43 -0
- truequery/standard/store/full-refresh-by-replace-not-delete-insert.yaml +44 -0
- truequery/standard/store/incremental-watermark-partition-alignment.yaml +40 -0
- truequery/standard/store/no-high-cardinality-partition-column.yaml +47 -0
- truequery/standard/store/partition-expiration-retention-policy.yaml +43 -0
- truequery/standard/store/partition-on-filter-column.yaml +43 -0
- truequery/standard/store/partition-predicate-required.yaml +41 -0
- truequery/standard/store/prefer-auto-liquid-clustering-over-manual-tune.yaml +37 -0
- truequery/standard/store/rewrite-manifests-and-stats-periodically.yaml +37 -0
- truequery/standard/store/single-open-format-per-medallion-layer.yaml +37 -0
- truequery/standard/store/time-travel-retention-explicit.yaml +46 -0
- truequery/standard/store/use-transient-temporary-for-ephemeral.yaml +35 -0
- truequery/standard/store/vacuum-expire-snapshots-with-retention-floor.yaml +42 -0
- truequery/std_registry.py +3310 -0
- truequery-0.1.0.dist-info/METADATA +52 -0
- truequery-0.1.0.dist-info/RECORD +254 -0
- truequery-0.1.0.dist-info/WHEEL +4 -0
- truequery-0.1.0.dist-info/entry_points.txt +3 -0
- truequery-0.1.0.dist-info/licenses/LICENSE +201 -0
- truequery-0.1.0.dist-info/licenses/NOTICE +4 -0
truequery/checks/sql.py
ADDED
|
@@ -0,0 +1,1278 @@
|
|
|
1
|
+
"""File-level SQL correctness checks, one function per standard record.
|
|
2
|
+
|
|
3
|
+
Each check takes a parsed statement, a Locator and a CheckContext
|
|
4
|
+
(dialect, path, mode, declared keys) and returns a list of
|
|
5
|
+
``(node, why, fix)`` tuples; the analyzer attaches rule slug, severity,
|
|
6
|
+
and location. Rules are keyed by standard slug — there is no second ID
|
|
7
|
+
scheme. Checks only fire where the query objectively returns a wrong
|
|
8
|
+
answer, loses data, or is non-deterministic.
|
|
9
|
+
"""
|
|
10
|
+
import re
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
|
|
13
|
+
from sqlglot import exp
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass(frozen=True)
|
|
17
|
+
class CheckContext:
|
|
18
|
+
"""What a check may know beyond the statement itself."""
|
|
19
|
+
dialect: str | None = None
|
|
20
|
+
path: str = "<input>"
|
|
21
|
+
mode: str = "build"
|
|
22
|
+
keys: dict = field(default_factory=dict) # relation -> candidate keys
|
|
23
|
+
|
|
24
|
+
# Functions whose result is non-NULL whenever a fallback is non-NULL:
|
|
25
|
+
# a nullable-side column wrapped in one is NULL-tolerant.
|
|
26
|
+
TOLERANT_FNS = frozenset({
|
|
27
|
+
"coalesce", "ifnull", "nvl", "nvl2", "if", "zeroifnull",
|
|
28
|
+
})
|
|
29
|
+
NONDETERMINISTIC_FNS = frozenset({
|
|
30
|
+
"uuid", "uuid_string", "rand", "random", "uniform", "randstr",
|
|
31
|
+
"now", "current_timestamp", "current_time", "current_date",
|
|
32
|
+
"sysdate", "getdate", "newid",
|
|
33
|
+
})
|
|
34
|
+
# Set-returning items whose rows come from the left side: joining one
|
|
35
|
+
# without ON is idiomatic, not cartesian.
|
|
36
|
+
_GENERATOR_NODES = (exp.Unnest, exp.GenerateSeries, exp.Explode, exp.Lateral)
|
|
37
|
+
_GENERATOR_FNS = frozenset({"unnest", "generate_series", "flatten",
|
|
38
|
+
"explode", "range"})
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _is_generator(item):
|
|
42
|
+
"""The FROM item is a set-returning function (UNNEST, FLATTEN,
|
|
43
|
+
generate_series ...), judged on the AST: a table that merely has
|
|
44
|
+
`range` in its name is still a table."""
|
|
45
|
+
for node in (item, getattr(item, "this", None)):
|
|
46
|
+
if isinstance(node, _GENERATOR_NODES):
|
|
47
|
+
return True
|
|
48
|
+
if isinstance(node, exp.Func) and node.sql_name().lower() in _GENERATOR_FNS:
|
|
49
|
+
return True
|
|
50
|
+
return False
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _join_side(join):
|
|
54
|
+
"""LEFT / RIGHT / FULL for outer joins, "" for inner and cross joins.
|
|
55
|
+
Read from the AST, never from the join's text (a column called
|
|
56
|
+
`left_id` in an ON clause does not make the join outer)."""
|
|
57
|
+
side = (join.args.get("side") or "").upper()
|
|
58
|
+
kind = (join.args.get("kind") or "").upper()
|
|
59
|
+
if kind in ("SEMI", "ANTI"):
|
|
60
|
+
return "" # LEFT SEMI/ANTI joins add no nullable columns
|
|
61
|
+
return side if side in ("LEFT", "RIGHT", "FULL") else ""
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _ident_key(ident):
|
|
65
|
+
"""Canonical key for one identifier, from the AST — never printed SQL.
|
|
66
|
+
|
|
67
|
+
Unquoted (and quoted all-lowercase) identifiers compare
|
|
68
|
+
case-insensitively ("U:o"); other quoted ones compare exactly ("Q:O"). Missing
|
|
69
|
+
identifiers key as "" (falsy, like the old sentinel)."""
|
|
70
|
+
if ident is None:
|
|
71
|
+
return ""
|
|
72
|
+
if isinstance(ident, exp.Identifier):
|
|
73
|
+
name = ident.name
|
|
74
|
+
# A quoted all-lowercase name is the same relation as the
|
|
75
|
+
# unquoted one ("orders" = orders); other quoted names are exact.
|
|
76
|
+
if ident.args.get("quoted") and name != name.lower():
|
|
77
|
+
return "Q:" + name
|
|
78
|
+
return "U:" + name.lower()
|
|
79
|
+
text = str(ident)
|
|
80
|
+
return "U:" + text.lower() if text else ""
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _col_key(col):
|
|
84
|
+
"""Canonical key for a column reference: qualifier + name."""
|
|
85
|
+
return _ident_key(col.args.get("table")) + "." + _ident_key(col.args.get("this"))
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _alias_key(node):
|
|
89
|
+
"""Alias name of a FROM item, without any column-alias list.
|
|
90
|
+
|
|
91
|
+
`g(d)` keys as the alias `g`, never the printed `g(d)`; quoted
|
|
92
|
+
aliases keep their exact case. Shared by every check that
|
|
93
|
+
resolves aliases."""
|
|
94
|
+
if node is None:
|
|
95
|
+
return ""
|
|
96
|
+
alias = node.args.get("alias") if hasattr(node, "args") else None
|
|
97
|
+
if alias is not None:
|
|
98
|
+
inner = alias.args.get("this")
|
|
99
|
+
if isinstance(inner, exp.Identifier):
|
|
100
|
+
return _ident_key(inner)
|
|
101
|
+
this = node.args.get("this") if hasattr(node, "args") else None
|
|
102
|
+
if isinstance(this, exp.Identifier):
|
|
103
|
+
return _ident_key(this)
|
|
104
|
+
name = node.name if hasattr(node, "name") else ""
|
|
105
|
+
return "U:" + name.lower() if name else ""
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _alias_display(node):
|
|
109
|
+
"""Human-readable alias for messages (no key prefixes)."""
|
|
110
|
+
name = getattr(node, "alias_or_name", "") or ""
|
|
111
|
+
return name or "?"
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
# -- sql/not-in-nullable-subquery --------------------------------------------
|
|
115
|
+
def _all_literal_projections(scope):
|
|
116
|
+
"""Every branch of the subquery projects a non-null literal (plain
|
|
117
|
+
SELECT 1, or UNION branches of literals). Literals are never NULL,
|
|
118
|
+
so NOT IN over them cannot be poisoned."""
|
|
119
|
+
selects = list(scope.find_all(exp.Select))
|
|
120
|
+
if not selects:
|
|
121
|
+
return False
|
|
122
|
+
for sel in selects:
|
|
123
|
+
proj = (sel.expressions or [None])[0]
|
|
124
|
+
col = proj.unalias() if hasattr(proj, "unalias") else proj
|
|
125
|
+
if not isinstance(col, exp.Literal):
|
|
126
|
+
return False
|
|
127
|
+
return True
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _eq_pins_non_null(eq, key):
|
|
131
|
+
"""An equality conjunct pins the column when it must hold TRUE: the
|
|
132
|
+
other side is a non-NULL literal or another column (NULL = anything
|
|
133
|
+
is unknown, so surviving rows have both sides non-null)."""
|
|
134
|
+
if not isinstance(eq, exp.EQ):
|
|
135
|
+
return False
|
|
136
|
+
left, right = eq.left, eq.right
|
|
137
|
+
if isinstance(left, exp.Column) and _col_key(left) in key:
|
|
138
|
+
other = right
|
|
139
|
+
elif isinstance(right, exp.Column) and _col_key(right) in key:
|
|
140
|
+
other = left
|
|
141
|
+
else:
|
|
142
|
+
return False
|
|
143
|
+
if isinstance(other, exp.Column):
|
|
144
|
+
return True
|
|
145
|
+
# Any literal pins (strings and numbers are values); NULL parses as
|
|
146
|
+
# exp.Null, never exp.Literal.
|
|
147
|
+
return isinstance(other, exp.Literal)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _sub_never_null(sel):
|
|
151
|
+
"""The subquery's projected column is provably non-null from the file:
|
|
152
|
+
a COUNT (never NULL), a WHERE equality / IS NOT NULL on it, or an
|
|
153
|
+
INNER-join equality on it (only rows with both sides non-null join)."""
|
|
154
|
+
proj = (sel.expressions or [None])[0]
|
|
155
|
+
col = proj.unalias() if hasattr(proj, "unalias") else proj
|
|
156
|
+
if isinstance(col, exp.Count):
|
|
157
|
+
return True
|
|
158
|
+
if not isinstance(col, exp.Column):
|
|
159
|
+
return False
|
|
160
|
+
# With a single FROM source, `o.customer_id` and `customer_id` are the
|
|
161
|
+
# same column: accept either spelling.
|
|
162
|
+
key = {_col_key(col)}
|
|
163
|
+
if not sel.args.get("joins"):
|
|
164
|
+
key |= {"." + _ident_key(col.args.get("this"))}
|
|
165
|
+
if col.args.get("table") is None:
|
|
166
|
+
frm = sel.args.get("from_")
|
|
167
|
+
if frm is not None and frm.this is not None:
|
|
168
|
+
key.add(_alias_key(frm.this) + "." + _ident_key(col.args.get("this")))
|
|
169
|
+
where = sel.args.get("where")
|
|
170
|
+
if where is not None:
|
|
171
|
+
for operand in _and_operands(where.this):
|
|
172
|
+
node = operand
|
|
173
|
+
while isinstance(node, exp.Paren):
|
|
174
|
+
node = node.this
|
|
175
|
+
if _is_not_null_test(node, key):
|
|
176
|
+
return True
|
|
177
|
+
if isinstance(node, exp.Not):
|
|
178
|
+
continue
|
|
179
|
+
if _eq_pins_non_null(node, key):
|
|
180
|
+
return True
|
|
181
|
+
for join in sel.args.get("joins", []):
|
|
182
|
+
# Plain JOIN and explicit INNER JOIN filter NULLs on equality;
|
|
183
|
+
# anything with a side (LEFT/RIGHT/FULL), a method (NATURAL) or
|
|
184
|
+
# a non-INNER kind (CROSS) does not.
|
|
185
|
+
method = (join.args.get("method") or "").upper()
|
|
186
|
+
kind = (join.args.get("kind") or "").upper()
|
|
187
|
+
side = (join.args.get("side") or "").upper()
|
|
188
|
+
if side or method not in ("", "INNER") or kind not in ("", "INNER"):
|
|
189
|
+
continue
|
|
190
|
+
on = join.args.get("on")
|
|
191
|
+
if on is None:
|
|
192
|
+
continue
|
|
193
|
+
for operand in _and_operands(on):
|
|
194
|
+
node = operand
|
|
195
|
+
while isinstance(node, exp.Paren):
|
|
196
|
+
node = node.this
|
|
197
|
+
if _eq_pins_non_null(node, key):
|
|
198
|
+
return True
|
|
199
|
+
return False
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _is_not_null_test(node, key):
|
|
203
|
+
"""`col IS NOT NULL` (sqlglot: NOT (col IS NULL)) on exactly this column."""
|
|
204
|
+
if not isinstance(node, exp.Not):
|
|
205
|
+
return False
|
|
206
|
+
inner = node.this
|
|
207
|
+
while isinstance(inner, exp.Paren):
|
|
208
|
+
inner = inner.this
|
|
209
|
+
return (isinstance(inner, exp.Is)
|
|
210
|
+
and isinstance(inner.this, exp.Column)
|
|
211
|
+
and _col_key(inner.this) in key
|
|
212
|
+
and isinstance(inner.expression, exp.Null))
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _set_branches(node):
|
|
216
|
+
"""The SELECT branches of a (possibly nested) set operation."""
|
|
217
|
+
if isinstance(node, exp.Subquery):
|
|
218
|
+
return _set_branches(node.this)
|
|
219
|
+
if isinstance(node, exp.SetOperation):
|
|
220
|
+
return _set_branches(node.this) + _set_branches(node.expression)
|
|
221
|
+
return [node] if isinstance(node, exp.Select) else []
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _sub_nullable(scope):
|
|
225
|
+
"""The subquery may yield NULL: some branch projects a column with no
|
|
226
|
+
guard that must hold for every surviving row (a top-level AND
|
|
227
|
+
conjunct, never one OR branch)."""
|
|
228
|
+
if _all_literal_projections(scope):
|
|
229
|
+
return False
|
|
230
|
+
branches = _set_branches(scope)
|
|
231
|
+
return not branches or any(not _sub_never_null(b) for b in branches)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def not_in_nullable_subquery(stmt, loc, ctx):
|
|
235
|
+
out = []
|
|
236
|
+
for not_n in stmt.find_all(exp.Not):
|
|
237
|
+
inner = not_n.this
|
|
238
|
+
while isinstance(inner, exp.Paren):
|
|
239
|
+
inner = inner.this
|
|
240
|
+
if not isinstance(inner, exp.In):
|
|
241
|
+
continue
|
|
242
|
+
sub = inner.args.get("query") or inner.find(exp.Subquery) \
|
|
243
|
+
or inner.find(exp.Select)
|
|
244
|
+
if sub is None or not _sub_nullable(sub):
|
|
245
|
+
continue
|
|
246
|
+
out.append((not_n,
|
|
247
|
+
"a NULL in the subquery makes NOT IN return zero rows",
|
|
248
|
+
"rewrite as NOT EXISTS, or add WHERE col IS NOT NULL"))
|
|
249
|
+
return out
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
# -- sql/unguarded-update-delete ----------------------------------------------
|
|
253
|
+
def _or_operands(node):
|
|
254
|
+
if isinstance(node, exp.Or):
|
|
255
|
+
yield from _or_operands(node.left)
|
|
256
|
+
yield from _or_operands(node.right)
|
|
257
|
+
else:
|
|
258
|
+
yield node
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def _is_tautology(cond):
|
|
262
|
+
while isinstance(cond, exp.Paren):
|
|
263
|
+
cond = cond.this
|
|
264
|
+
if isinstance(cond, exp.Boolean) and cond.this is True:
|
|
265
|
+
return True
|
|
266
|
+
if isinstance(cond, exp.Or):
|
|
267
|
+
# X OR TRUE is TRUE (three-valued logic: TRUE absorbs anything)
|
|
268
|
+
if any(_is_tautology(branch) for branch in _or_operands(cond)):
|
|
269
|
+
return True
|
|
270
|
+
if isinstance(cond, exp.EQ):
|
|
271
|
+
left, right = cond.left, cond.right
|
|
272
|
+
if isinstance(left, exp.Literal) and isinstance(right, exp.Literal):
|
|
273
|
+
return left.name == right.name
|
|
274
|
+
return False
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def _under_merge(node):
|
|
278
|
+
parent = getattr(node, "parent", None)
|
|
279
|
+
while parent is not None:
|
|
280
|
+
if isinstance(parent, exp.Merge):
|
|
281
|
+
return True
|
|
282
|
+
parent = getattr(parent, "parent", None)
|
|
283
|
+
return False
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def _dml_sources(node):
|
|
287
|
+
"""Extra source tables of cartesian DML: DELETE .. USING, UPDATE .. FROM."""
|
|
288
|
+
sources = []
|
|
289
|
+
if isinstance(node, exp.Delete):
|
|
290
|
+
for table in node.args.get("using") or []:
|
|
291
|
+
sources.append(table)
|
|
292
|
+
if isinstance(node, exp.Update):
|
|
293
|
+
frm = node.args.get("from_")
|
|
294
|
+
if frm is not None:
|
|
295
|
+
sources.append(frm.this)
|
|
296
|
+
return [t for t in sources if t is not None]
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
def _is_down_migration(path):
|
|
300
|
+
"""Down/rollback migrations intentionally reset whole tables, so a
|
|
301
|
+
missing WHERE is the intent, not an accident. Matched on whole path
|
|
302
|
+
names (`000157_x.down.sql`, `x_rollback.sql`, a `down/` directory),
|
|
303
|
+
so `markdown/` and `drill-down.sql` do not count. Change mode alone is not enough (up-migrations in
|
|
304
|
+
Change mode still get checked)."""
|
|
305
|
+
parts = (path or "").replace("\\", "/").lower().split("/")
|
|
306
|
+
if set(parts[:-1]) & {"down", "rollback", "rollbacks"}:
|
|
307
|
+
return True
|
|
308
|
+
return bool(re.search(r"(^|[._])(down|rollback)\.sql$", parts[-1]))
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def unguarded_update_delete(stmt, loc, ctx):
|
|
312
|
+
if _is_down_migration(ctx.path):
|
|
313
|
+
return []
|
|
314
|
+
out = []
|
|
315
|
+
for node in list(stmt.find_all(exp.Delete)) + list(stmt.find_all(exp.Update)):
|
|
316
|
+
if isinstance(node, exp.Update) and _under_merge(node):
|
|
317
|
+
continue # MERGE ... WHEN MATCHED THEN UPDATE is guarded by ON
|
|
318
|
+
where = node.args.get("where")
|
|
319
|
+
if where is None:
|
|
320
|
+
out.append((node, "no WHERE: every row is affected",
|
|
321
|
+
"add a WHERE predicate, or use TRUNCATE if you mean the whole table"))
|
|
322
|
+
elif _is_tautology(where.this):
|
|
323
|
+
out.append((node, "WHERE is a constant-true tautology",
|
|
324
|
+
"replace with a real predicate"))
|
|
325
|
+
else:
|
|
326
|
+
# Cartesian DML: the target is never linked to the source,
|
|
327
|
+
# so the write hits every row (or every filtered-source row).
|
|
328
|
+
target = _alias_key(node.this) if node.this is not None else ""
|
|
329
|
+
for source in _dml_sources(node):
|
|
330
|
+
name = _alias_key(source)
|
|
331
|
+
if not target or not name or name == target:
|
|
332
|
+
continue
|
|
333
|
+
# An unqualified column can only be the target's (or is
|
|
334
|
+
# ambiguous and the engine says so): it links too.
|
|
335
|
+
linked = any(name in _qualifiers(op)
|
|
336
|
+
and (target in _qualifiers(op)
|
|
337
|
+
or _links_unqualified(op))
|
|
338
|
+
for op in _and_operands(where.this))
|
|
339
|
+
if not linked:
|
|
340
|
+
src, tgt = _alias_display(source), _alias_display(node.this)
|
|
341
|
+
out.append((node, f"no predicate links {src} to {tgt}: "
|
|
342
|
+
"the write is cartesian",
|
|
343
|
+
f"link them (WHERE {tgt}.<key> = {src}.<key>) "
|
|
344
|
+
"or drop the unused source"))
|
|
345
|
+
break
|
|
346
|
+
return out
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
# -- sql/accidental-cross-join --------------------------------------------------
|
|
350
|
+
def _same_column_eq(cond):
|
|
351
|
+
"""`x = x` (same qualifier and name, modulo folding) — a tautology
|
|
352
|
+
by meaning. Shared with the ON-tautology check below."""
|
|
353
|
+
node = cond
|
|
354
|
+
while isinstance(node, exp.Paren):
|
|
355
|
+
node = node.this
|
|
356
|
+
if not isinstance(node, exp.EQ):
|
|
357
|
+
return False
|
|
358
|
+
if not (isinstance(node.left, exp.Column) and isinstance(node.right, exp.Column)):
|
|
359
|
+
return False
|
|
360
|
+
return _col_key(node.left) == _col_key(node.right)
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def accidental_cross_join(stmt, loc, ctx):
|
|
364
|
+
out = []
|
|
365
|
+
for join in stmt.find_all(exp.Join):
|
|
366
|
+
if _is_generator(join.this):
|
|
367
|
+
continue
|
|
368
|
+
if "__tq_jinja" in join.sql().lower():
|
|
369
|
+
continue # a dbt macro expands here: the join is unknowable
|
|
370
|
+
# from the file, and no finding goes out without evidence
|
|
371
|
+
method = (join.args.get("method") or "").upper()
|
|
372
|
+
kind = (join.args.get("kind") or "").upper()
|
|
373
|
+
if (method == "CROSS" or kind == "CROSS") and _written_cross(join, loc):
|
|
374
|
+
continue # an explicit CROSS JOIN is intentional
|
|
375
|
+
if method == "NATURAL" or kind == "NATURAL":
|
|
376
|
+
continue # owned by sql/natural-join-explicit-keys
|
|
377
|
+
if join.args.get("using"):
|
|
378
|
+
continue # JOIN ... USING names explicit keys
|
|
379
|
+
right_alias = _alias_key(join.this)
|
|
380
|
+
sel = _owning_select(join)
|
|
381
|
+
others = _from_keys(sel, join, right_alias) if sel is not None else set()
|
|
382
|
+
if _is_correlated(join.this, others, ctx.dialect):
|
|
383
|
+
continue # LATERAL / function over earlier columns: linked
|
|
384
|
+
constraint = join.args.get("on")
|
|
385
|
+
if constraint is not None:
|
|
386
|
+
if _is_tautology(constraint) or _same_column_eq(constraint):
|
|
387
|
+
out.append((join, "join condition is always true",
|
|
388
|
+
"add the real join predicate or use CROSS JOIN explicitly"))
|
|
389
|
+
continue
|
|
390
|
+
if _on_links_both_sides(join, _alias_key(join.this)):
|
|
391
|
+
continue
|
|
392
|
+
out.append((join, "join predicate references only one side: "
|
|
393
|
+
"no linking predicate (cartesian)",
|
|
394
|
+
"link both sides with ON <keys>, or use CROSS JOIN explicitly"))
|
|
395
|
+
continue
|
|
396
|
+
this_alias = _alias_key(join.this)
|
|
397
|
+
if not _joined_on_ast(join, this_alias):
|
|
398
|
+
out.append((join, f"no predicate links {_alias_display(join.this)}: "
|
|
399
|
+
"every row pairs with every row on the other side",
|
|
400
|
+
"add ON <keys>, or CROSS JOIN if cartesian is intended"))
|
|
401
|
+
return out
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
_JOIN_WORD = re.compile(r",|\bJOIN\b", re.IGNORECASE)
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def _written_cross(join, loc):
|
|
408
|
+
"""The join was written as CROSS JOIN, not as a comma. Several
|
|
409
|
+
dialects (BigQuery, Spark, SQLite, ClickHouse) parse `FROM a, b`
|
|
410
|
+
into the same tree as `a CROSS JOIN b`, so read the source text
|
|
411
|
+
just before the joined item. Unknown position: trust the tree."""
|
|
412
|
+
starts = [n.meta["start"] for n in join.this.walk()
|
|
413
|
+
if isinstance(n, exp.Identifier)
|
|
414
|
+
and isinstance(n.meta.get("start"), int)]
|
|
415
|
+
if not starts:
|
|
416
|
+
return True
|
|
417
|
+
before = loc.text[:min(starts)]
|
|
418
|
+
hits = list(_JOIN_WORD.finditer(before))
|
|
419
|
+
if not hits:
|
|
420
|
+
return True
|
|
421
|
+
last = hits[-1]
|
|
422
|
+
if last.group(0) == ",":
|
|
423
|
+
return False
|
|
424
|
+
return bool(re.search(r"\bCROSS\s+$", before[:last.start()],
|
|
425
|
+
re.IGNORECASE))
|
|
426
|
+
|
|
427
|
+
|
|
428
|
+
def _and_operands(node):
|
|
429
|
+
if isinstance(node, exp.And):
|
|
430
|
+
yield from _and_operands(node.left)
|
|
431
|
+
yield from _and_operands(node.right)
|
|
432
|
+
else:
|
|
433
|
+
yield node
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def _links_unqualified(node):
|
|
437
|
+
"""A predicate comparing two or more columns where one is
|
|
438
|
+
unqualified (`ON customer_id = c.id`): the unqualified one resolves
|
|
439
|
+
against the other FROM items, so it can link them."""
|
|
440
|
+
cols = list(node.find_all(exp.Column))
|
|
441
|
+
return len(cols) >= 2 and any(c.args.get("table") is None for c in cols)
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
def _qualifiers(node):
|
|
445
|
+
return {_ident_key(c.args.get("table")) for c in node.find_all(exp.Column)}
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
def _from_keys(sel, join, this_alias):
|
|
449
|
+
"""Alias keys of every top-level FROM item except this one."""
|
|
450
|
+
others = set()
|
|
451
|
+
frm = sel.args.get("from_")
|
|
452
|
+
if frm is not None and frm.this is not None:
|
|
453
|
+
# Top-level FROM item only (a derived table keys as its alias,
|
|
454
|
+
# never the tables nested inside it).
|
|
455
|
+
name = _alias_key(frm.this)
|
|
456
|
+
if name and name != this_alias:
|
|
457
|
+
others.add(name)
|
|
458
|
+
for other in sel.args.get("joins", []):
|
|
459
|
+
name = _alias_key(other.this) if other is not join else ""
|
|
460
|
+
if name and name != this_alias:
|
|
461
|
+
others.add(name)
|
|
462
|
+
return others
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
def _joined_on_ast(join, this_alias):
|
|
466
|
+
"""True when some predicate links this table to another FROM table,
|
|
467
|
+
judged on column qualifiers — never on text substrings."""
|
|
468
|
+
sel = _owning_select(join)
|
|
469
|
+
if sel is None:
|
|
470
|
+
return False
|
|
471
|
+
others = _from_keys(sel, join, this_alias)
|
|
472
|
+
if not this_alias or not others:
|
|
473
|
+
return False
|
|
474
|
+
where = sel.args.get("where")
|
|
475
|
+
if where is None:
|
|
476
|
+
return False
|
|
477
|
+
for operand in _and_operands(where.this):
|
|
478
|
+
quals = _level_qualifiers(operand)
|
|
479
|
+
if (this_alias in quals and quals & others) or (
|
|
480
|
+
"" in quals and _links_unqualified(operand)):
|
|
481
|
+
return True
|
|
482
|
+
return False
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
def _is_correlated(item, others, dialect=None):
|
|
486
|
+
"""The joined item reads columns of earlier FROM items (LATERAL
|
|
487
|
+
subquery, UNNEST(a.col), table function over a column). That
|
|
488
|
+
correlation IS the link, so ON TRUE is idiomatic, not cartesian."""
|
|
489
|
+
if not others:
|
|
490
|
+
return False
|
|
491
|
+
# BigQuery-style implicit UNNEST: `FROM t, t.array_col AS x` names a
|
|
492
|
+
# column path of an earlier FROM item, so it is correlated too. Only
|
|
493
|
+
# for BigQuery or an unknown dialect: elsewhere `sales.customers` next
|
|
494
|
+
# to an alias `sales` is a schema-qualified table.
|
|
495
|
+
if (dialect in (None, "bigquery") and isinstance(item, exp.Table)
|
|
496
|
+
and _ident_key(item.args.get("db")) in others):
|
|
497
|
+
return True
|
|
498
|
+
return any(_ident_key(col.args.get("table")) in others
|
|
499
|
+
for col in item.find_all(exp.Column))
|
|
500
|
+
|
|
501
|
+
|
|
502
|
+
def _level_qualifiers(node):
|
|
503
|
+
"""Qualifier keys of columns at the node's own query level. Columns
|
|
504
|
+
inside nested subqueries live in a different scope: their qualifiers
|
|
505
|
+
(or lack of one) say nothing about how the outer tables link, so
|
|
506
|
+
`ON x.id NOT IN (SELECT uid ...)` never links the outer table."""
|
|
507
|
+
quals = set()
|
|
508
|
+
for col in node.find_all(exp.Column):
|
|
509
|
+
probe = col
|
|
510
|
+
nested = False
|
|
511
|
+
while probe is not node and probe is not None:
|
|
512
|
+
probe = getattr(probe, "parent", None)
|
|
513
|
+
if isinstance(probe, exp.Select):
|
|
514
|
+
nested = True
|
|
515
|
+
break
|
|
516
|
+
if not nested:
|
|
517
|
+
quals.add(_ident_key(col.args.get("table")))
|
|
518
|
+
return quals
|
|
519
|
+
|
|
520
|
+
|
|
521
|
+
def _on_links_both_sides(join, right_alias):
|
|
522
|
+
"""An ON clause links the joined table if some AND-branch references
|
|
523
|
+
it with a real predicate. The other side may be unqualified
|
|
524
|
+
(`ON key = t.key` can only resolve against the remaining FROM scope),
|
|
525
|
+
so requiring both qualifiers misfires on ordinary joins. `ON a.flag
|
|
526
|
+
= 1` still crosses: it never references the joined table at all."""
|
|
527
|
+
sel = _owning_select(join)
|
|
528
|
+
if sel is None or not right_alias:
|
|
529
|
+
return False
|
|
530
|
+
others = _from_keys(sel, join, right_alias)
|
|
531
|
+
if not others:
|
|
532
|
+
return False
|
|
533
|
+
on = join.args.get("on")
|
|
534
|
+
if on is None:
|
|
535
|
+
return False
|
|
536
|
+
for operand in _and_operands(on):
|
|
537
|
+
quals = _level_qualifiers(operand)
|
|
538
|
+
if right_alias in quals and (quals & others or "" in quals):
|
|
539
|
+
return True
|
|
540
|
+
if quals == {""} and _links_unqualified(operand):
|
|
541
|
+
return True # ON customer_id = cust_id
|
|
542
|
+
return False
|
|
543
|
+
|
|
544
|
+
|
|
545
|
+
# -- sql/left-join-null-rejecting-where -----------------------------------------
|
|
546
|
+
# General rule: any predicate on a nullable-side column that is
|
|
547
|
+
# NULL-rejecting counts — comparisons, BETWEEN, LIKE/ILIKE, IN, and
|
|
548
|
+
# non-tolerant function calls. NULL-tolerant: IS NULL (anti-join),
|
|
549
|
+
# IS NOT NULL (explicit), COALESCE/IFNULL/NVL wrappers, OR ... IS NULL,
|
|
550
|
+
# IS NOT DISTINCT FROM.
|
|
551
|
+
def _nullable_aliases(join, left_alias):
|
|
552
|
+
side = _join_side(join)
|
|
553
|
+
right = _alias_key(join.this)
|
|
554
|
+
if side == "LEFT":
|
|
555
|
+
return {right} - {""}
|
|
556
|
+
if side == "RIGHT":
|
|
557
|
+
return {left_alias} - {""}
|
|
558
|
+
if side == "FULL":
|
|
559
|
+
return {left_alias, right} - {""}
|
|
560
|
+
return set()
|
|
561
|
+
|
|
562
|
+
|
|
563
|
+
def _side_exposed(side, nullable):
|
|
564
|
+
if isinstance(side, exp.Column):
|
|
565
|
+
return _ident_key(side.args.get("table")) in nullable
|
|
566
|
+
if isinstance(side, exp.Func):
|
|
567
|
+
if side.sql_name().lower() in TOLERANT_FNS:
|
|
568
|
+
return False
|
|
569
|
+
return any(_side_exposed(a, nullable) for a in side.args.values()
|
|
570
|
+
if isinstance(a, exp.Expression))
|
|
571
|
+
if isinstance(side, (exp.Paren, exp.Cast, exp.Neg)):
|
|
572
|
+
kids = side.args.get("this")
|
|
573
|
+
return _side_exposed(kids, nullable) if isinstance(kids, exp.Expression) else False
|
|
574
|
+
if isinstance(side, exp.Binary):
|
|
575
|
+
return _side_exposed(side.left, nullable) or _side_exposed(side.right, nullable)
|
|
576
|
+
return False
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
_COMPARISONS = (exp.EQ, exp.GT, exp.GTE, exp.LT, exp.LTE, exp.NEQ)
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
def _pred_rejects(pred, nullable):
|
|
583
|
+
"""True when pred is NULL-rejecting on a nullable-side column."""
|
|
584
|
+
if isinstance(pred, exp.Is):
|
|
585
|
+
return False # IS NULL (anti-join) and IS NOT NULL (explicit) are intentional
|
|
586
|
+
if isinstance(pred, exp.NullSafeEQ):
|
|
587
|
+
# Null-tolerant between two values — but `col IS NOT DISTINCT
|
|
588
|
+
# FROM <non-null literal>` is false for NULL, so it rejects
|
|
589
|
+
# NULLs exactly like `=` and kills the outer join.
|
|
590
|
+
sides = [pred.args.get(k) for k in
|
|
591
|
+
("this", "expression", "left", "right")]
|
|
592
|
+
sides = [s for s in sides
|
|
593
|
+
if isinstance(s, exp.Expression)]
|
|
594
|
+
exposed = any(isinstance(s, exp.Column)
|
|
595
|
+
and _ident_key(s.args.get("table")) in nullable
|
|
596
|
+
for s in sides)
|
|
597
|
+
literal = any(isinstance(s, exp.Literal) for s in sides)
|
|
598
|
+
return exposed and literal
|
|
599
|
+
if isinstance(pred, _COMPARISONS):
|
|
600
|
+
return (_side_exposed(pred.left, nullable)
|
|
601
|
+
or _side_exposed(pred.right, nullable))
|
|
602
|
+
if isinstance(pred, exp.Between):
|
|
603
|
+
return _side_exposed(pred.args.get("this"), nullable)
|
|
604
|
+
if isinstance(pred, (exp.Like, exp.ILike)):
|
|
605
|
+
return _side_exposed(pred.args.get("this"), nullable)
|
|
606
|
+
if isinstance(pred, exp.In):
|
|
607
|
+
if pred.args.get("not_") and (pred.find(exp.Subquery) or pred.find(exp.Select)):
|
|
608
|
+
return False # owned by sql/not-in-nullable-subquery
|
|
609
|
+
return _side_exposed(pred.args.get("this"), nullable)
|
|
610
|
+
if isinstance(pred, exp.Func):
|
|
611
|
+
if isinstance(pred, exp.Connector):
|
|
612
|
+
return False # AND/OR structure, not a function call
|
|
613
|
+
if pred.sql_name().lower() in TOLERANT_FNS:
|
|
614
|
+
return False
|
|
615
|
+
return any(_side_exposed(a, nullable) for a in pred.args.values()
|
|
616
|
+
if isinstance(a, exp.Expression))
|
|
617
|
+
return False
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
def left_join_null_rejecting_where(stmt, loc, ctx):
|
|
621
|
+
out = []
|
|
622
|
+
for sel in stmt.find_all(exp.Select):
|
|
623
|
+
joins = sel.args.get("joins", [])
|
|
624
|
+
if not joins or sel.args.get("where") is None:
|
|
625
|
+
continue
|
|
626
|
+
where = sel.args.get("where")
|
|
627
|
+
frm = sel.args.get("from_")
|
|
628
|
+
left_alias = _alias_key(frm.this) if frm is not None and frm.this is not None else ""
|
|
629
|
+
nullable = set()
|
|
630
|
+
for join in joins:
|
|
631
|
+
nullable |= _nullable_aliases(join, left_alias)
|
|
632
|
+
if not nullable:
|
|
633
|
+
continue
|
|
634
|
+
for pred in where.find_all(*_COMPARISONS, exp.Between, exp.Like,
|
|
635
|
+
exp.ILike, exp.In, exp.Func,
|
|
636
|
+
exp.NullSafeEQ):
|
|
637
|
+
if _owning_select(pred) is not sel:
|
|
638
|
+
continue # a subquery (EXISTS ...) has its own scope
|
|
639
|
+
if _under_tolerant(pred, where):
|
|
640
|
+
continue # COALESCE(LOWER(o.x), 'none'): NULL handled
|
|
641
|
+
if not _pred_rejects(pred, nullable):
|
|
642
|
+
continue
|
|
643
|
+
if _or_null_tolerant(pred, where):
|
|
644
|
+
continue
|
|
645
|
+
out.append((pred,
|
|
646
|
+
"WHERE rejects NULLs on the nullable side: "
|
|
647
|
+
"outer join silently becomes inner",
|
|
648
|
+
"move the filter into the ON clause to keep "
|
|
649
|
+
"unmatched rows (or add OR right.key IS NULL: test "
|
|
650
|
+
"the join key, not the filtered column)"))
|
|
651
|
+
break
|
|
652
|
+
return out
|
|
653
|
+
|
|
654
|
+
|
|
655
|
+
def _under_tolerant(pred, where):
|
|
656
|
+
node = pred.parent
|
|
657
|
+
while node is not None and node is not where:
|
|
658
|
+
if isinstance(node, exp.Func) and node.sql_name().lower() in TOLERANT_FNS:
|
|
659
|
+
return True
|
|
660
|
+
node = getattr(node, "parent", None)
|
|
661
|
+
return False
|
|
662
|
+
|
|
663
|
+
|
|
664
|
+
def _or_null_tolerant(pred, where):
|
|
665
|
+
node = pred.parent
|
|
666
|
+
while node is not None and node is not where:
|
|
667
|
+
if isinstance(node, exp.Or):
|
|
668
|
+
return node.find(exp.Is) is not None
|
|
669
|
+
node = getattr(node, "parent", None)
|
|
670
|
+
return False
|
|
671
|
+
|
|
672
|
+
|
|
673
|
+
# -- sql/null-equality-comparison --------------------------------------------------
|
|
674
|
+
def _under_dml_write(pred):
|
|
675
|
+
node = pred.parent
|
|
676
|
+
child = pred
|
|
677
|
+
while node is not None:
|
|
678
|
+
if isinstance(node, (exp.Update, exp.OnConflict)):
|
|
679
|
+
return True # SET assignments (incl. upserts) are not comparisons
|
|
680
|
+
if isinstance(node, exp.Merge):
|
|
681
|
+
# SET assignments are; the ON clause is a real comparison
|
|
682
|
+
return child is not node.args.get("on")
|
|
683
|
+
if isinstance(node, (exp.Select, exp.Where)):
|
|
684
|
+
return False
|
|
685
|
+
child = node
|
|
686
|
+
node = getattr(node, "parent", None)
|
|
687
|
+
return False
|
|
688
|
+
|
|
689
|
+
|
|
690
|
+
def null_equality_comparison(stmt, loc, ctx):
|
|
691
|
+
out = []
|
|
692
|
+
for pred in list(stmt.find_all(exp.EQ)) + list(stmt.find_all(exp.NEQ)):
|
|
693
|
+
if _under_dml_write(pred):
|
|
694
|
+
continue # SET assignments (UPDATE/MERGE) are not comparisons
|
|
695
|
+
if any(isinstance(x, exp.Null) for x in (pred.left, pred.right)):
|
|
696
|
+
out.append((pred, "= NULL is never true (use IS NULL)",
|
|
697
|
+
"rewrite as IS NULL / IS NOT NULL"))
|
|
698
|
+
for case in stmt.find_all(exp.Case):
|
|
699
|
+
# Simple CASE uses equality: WHEN NULL never matches. Searched
|
|
700
|
+
# CASE (no operand) with WHEN x IS NULL is the fix, not a hit.
|
|
701
|
+
if case.args.get("this") is None:
|
|
702
|
+
continue
|
|
703
|
+
for cond in case.args.get("ifs", []):
|
|
704
|
+
if isinstance(cond.this, exp.Null):
|
|
705
|
+
out.append((cond, "WHEN NULL never matches (CASE uses "
|
|
706
|
+
"equality, and NULL = anything is unknown)",
|
|
707
|
+
"use a searched CASE with WHEN x IS NULL"))
|
|
708
|
+
break
|
|
709
|
+
return out
|
|
710
|
+
|
|
711
|
+
|
|
712
|
+
# -- sql/count-star-after-left-join --------------------------------------------------
|
|
713
|
+
def _outer_joins(sel):
|
|
714
|
+
return [join for join in sel.args.get("joins", []) if _join_side(join)]
|
|
715
|
+
|
|
716
|
+
|
|
717
|
+
def _cte_source(stmt, name):
|
|
718
|
+
"""Base table of the CTE called name (lowercased) when the CTE keeps
|
|
719
|
+
that table's grain (one FROM table, no joins, no GROUP BY), else None.
|
|
720
|
+
Searches the whole statement tree: files often parse as one large
|
|
721
|
+
statement with CTEs nested at several levels."""
|
|
722
|
+
for cte in stmt.find_all(exp.CTE):
|
|
723
|
+
if _alias_key(cte).split(":", 1)[-1] != name:
|
|
724
|
+
continue
|
|
725
|
+
inner = cte.this
|
|
726
|
+
if not isinstance(inner, exp.Select):
|
|
727
|
+
return None
|
|
728
|
+
if inner.args.get("joins") or inner.args.get("group"):
|
|
729
|
+
return None # may fan out or aggregate: not the base's grain
|
|
730
|
+
from_ = inner.args.get("from_")
|
|
731
|
+
table = from_.this if from_ is not None else None
|
|
732
|
+
if isinstance(table, exp.Table) and table.name:
|
|
733
|
+
return ".".join(p for p in (table.db, table.name) if p).lower()
|
|
734
|
+
return None
|
|
735
|
+
|
|
736
|
+
|
|
737
|
+
def _rhs_base_table(join, stmt):
|
|
738
|
+
"""Base table name behind a join's right side, lowercased. A table
|
|
739
|
+
resolves by its own name (never its alias: `customers c` is
|
|
740
|
+
customers), schema-qualified when written so; a name that is a CTE
|
|
741
|
+
resolves to the CTE's source table. A derived table resolves through
|
|
742
|
+
its alias only when that names a CTE."""
|
|
743
|
+
rhs = join.this
|
|
744
|
+
if rhs is None:
|
|
745
|
+
return ""
|
|
746
|
+
if isinstance(rhs, exp.Table) and rhs.name:
|
|
747
|
+
name = ".".join(p for p in (rhs.db, rhs.name) if p).lower()
|
|
748
|
+
return _cte_source(stmt, rhs.name.lower()) or name
|
|
749
|
+
name = _alias_key(rhs).split(":", 1)[-1]
|
|
750
|
+
return (_cte_source(stmt, name) or "") if name else ""
|
|
751
|
+
|
|
752
|
+
|
|
753
|
+
def _on_columns_for(join, rhs_alias):
|
|
754
|
+
"""Nullable-side columns constraining the join to at most one row:
|
|
755
|
+
the USING list, plus equalities that are top-level AND conjuncts of
|
|
756
|
+
the ON predicate. An OR anywhere at the top level (or a bare OR
|
|
757
|
+
predicate) matches several rows, so it contributes nothing."""
|
|
758
|
+
cols = set()
|
|
759
|
+
using = join.args.get("using") or []
|
|
760
|
+
for ident in using:
|
|
761
|
+
cols.add(ident.name.lower() if hasattr(ident, "name") else "")
|
|
762
|
+
on = join.args.get("on")
|
|
763
|
+
if on is None:
|
|
764
|
+
return cols - {""}
|
|
765
|
+
if isinstance(on, exp.Or):
|
|
766
|
+
return cols - {""}
|
|
767
|
+
for conj in _and_operands(on):
|
|
768
|
+
if not isinstance(conj, exp.EQ):
|
|
769
|
+
continue
|
|
770
|
+
for col in conj.find_all(exp.Column):
|
|
771
|
+
table = col.args.get("table")
|
|
772
|
+
tkey = _ident_key(table) if table is not None else ""
|
|
773
|
+
aname = tkey.split(":", 1)[-1]
|
|
774
|
+
if aname == rhs_alias:
|
|
775
|
+
cols.add(col.name.lower())
|
|
776
|
+
return cols - {""}
|
|
777
|
+
|
|
778
|
+
|
|
779
|
+
def _is_lookup_join(join, stmt, keys):
|
|
780
|
+
"""The ON predicate covers one declared candidate key of the nullable
|
|
781
|
+
side, so the join matches at most one row and COUNT(*) stays correct."""
|
|
782
|
+
rhs = join.this
|
|
783
|
+
if rhs is None:
|
|
784
|
+
return False
|
|
785
|
+
alias = _alias_key(rhs).split(":", 1)[-1]
|
|
786
|
+
base = _rhs_base_table(join, stmt)
|
|
787
|
+
if not base:
|
|
788
|
+
return False
|
|
789
|
+
# Plain and schema-qualified spellings share one entry: contracts
|
|
790
|
+
# declare `fx_rates`, SQL may say `raw.fx_rates`.
|
|
791
|
+
candidates = keys.get(base, []) + keys.get(base.split(".")[-1], [])
|
|
792
|
+
covered = _on_columns_for(join, alias)
|
|
793
|
+
return any(key <= covered for key in candidates)
|
|
794
|
+
|
|
795
|
+
|
|
796
|
+
def _owning_select(node):
|
|
797
|
+
"""Nearest enclosing Select, or None. Query-level attribution: a
|
|
798
|
+
COUNT(*) belongs to the SELECT it is written in, not to a sibling
|
|
799
|
+
CTE or an outer query."""
|
|
800
|
+
node = getattr(node, "parent", None)
|
|
801
|
+
while node is not None:
|
|
802
|
+
if isinstance(node, exp.Select):
|
|
803
|
+
return node
|
|
804
|
+
node = getattr(node, "parent", None)
|
|
805
|
+
return None
|
|
806
|
+
|
|
807
|
+
|
|
808
|
+
def count_star_after_left_join(stmt, loc, ctx):
|
|
809
|
+
keys = ctx.keys
|
|
810
|
+
out = []
|
|
811
|
+
seen = set()
|
|
812
|
+
for count in stmt.find_all(exp.Count):
|
|
813
|
+
# NOTE: grouped counts fire too (the record's proof is grouped:
|
|
814
|
+
# childless groups report 1, not 0).
|
|
815
|
+
arg = count.this
|
|
816
|
+
if not (isinstance(arg, exp.Star) or (
|
|
817
|
+
isinstance(arg, exp.Literal) and arg.name in ("1",))):
|
|
818
|
+
continue
|
|
819
|
+
# Same query level only: a COUNT(*) in an unjoined pre-agg CTE
|
|
820
|
+
# must not fire for a LEFT JOIN in the outer query, and vice versa.
|
|
821
|
+
sel = _owning_select(count)
|
|
822
|
+
if sel is None or not _outer_joins(sel):
|
|
823
|
+
continue
|
|
824
|
+
if id(sel) in seen:
|
|
825
|
+
continue
|
|
826
|
+
seen.add(id(sel))
|
|
827
|
+
# Lookup suppression: when every outer join at this level
|
|
828
|
+
# matches at most one row (ON covers a declared key of the
|
|
829
|
+
# nullable side), COUNT(*) counts preserved rows correctly.
|
|
830
|
+
outer = _outer_joins(sel)
|
|
831
|
+
if keys and outer and all(
|
|
832
|
+
_is_lookup_join(join, stmt, keys) for join in outer):
|
|
833
|
+
loc.suppressed.append("sql/count-star-after-left-join")
|
|
834
|
+
continue
|
|
835
|
+
alias = _alias_display(outer[0].this) if outer else "child"
|
|
836
|
+
out.append((count,
|
|
837
|
+
"COUNT(*) over an outer join counts 1 for unmatched rows",
|
|
838
|
+
f"count the nullable-side key instead, e.g. "
|
|
839
|
+
f"COUNT({alias}.<key>)"))
|
|
840
|
+
return out
|
|
841
|
+
|
|
842
|
+
|
|
843
|
+
# -- sql/nondeterministic-merge-keys --------------------------------------------------
|
|
844
|
+
def nondeterministic_merge_keys(stmt, loc, ctx):
|
|
845
|
+
out = []
|
|
846
|
+
for merge in stmt.find_all(exp.Merge):
|
|
847
|
+
on = merge.args.get("on")
|
|
848
|
+
if on is None:
|
|
849
|
+
continue
|
|
850
|
+
fired = False
|
|
851
|
+
for func in on.find_all(exp.Func):
|
|
852
|
+
if func.sql_name().lower() in NONDETERMINISTIC_FNS:
|
|
853
|
+
out.append((func,
|
|
854
|
+
"MERGE key uses a nondeterministic function: re-runs match differently",
|
|
855
|
+
"key the MERGE on stable source columns"))
|
|
856
|
+
fired = True
|
|
857
|
+
break
|
|
858
|
+
if fired:
|
|
859
|
+
continue
|
|
860
|
+
# nondeterminism can hide one level down: USING (SELECT uuid() AS
|
|
861
|
+
# uid ...) with ON comparing that alias.
|
|
862
|
+
using = merge.args.get("using")
|
|
863
|
+
if using is None:
|
|
864
|
+
continue
|
|
865
|
+
volatile = set()
|
|
866
|
+
for sel in using.find_all(exp.Select):
|
|
867
|
+
for proj in sel.expressions:
|
|
868
|
+
alias = proj.args.get("alias")
|
|
869
|
+
if alias is not None and proj.find(exp.Func) is not None and any(
|
|
870
|
+
f.sql_name().lower() in NONDETERMINISTIC_FNS
|
|
871
|
+
for f in proj.find_all(exp.Func)):
|
|
872
|
+
volatile.add(alias.name.lower())
|
|
873
|
+
if not volatile:
|
|
874
|
+
continue
|
|
875
|
+
for col in on.find_all(exp.Column):
|
|
876
|
+
if col.name.lower() in volatile:
|
|
877
|
+
out.append((col,
|
|
878
|
+
"MERGE key derives from a nondeterministic function: "
|
|
879
|
+
"re-runs match differently",
|
|
880
|
+
"key the MERGE on stable source columns"))
|
|
881
|
+
break
|
|
882
|
+
return out
|
|
883
|
+
|
|
884
|
+
|
|
885
|
+
# -- sql/set-operation-column-position --------------------------------------------------
|
|
886
|
+
def _branch_projection_star(branch):
|
|
887
|
+
"""True when a set-operation branch projects a bare `*` (SELECT * or
|
|
888
|
+
t.*) in its select list. Stars nested inside expressions — COUNT(*),
|
|
889
|
+
f(*) — say nothing about positional fragility."""
|
|
890
|
+
selects = [branch] if isinstance(branch, exp.Select) else (
|
|
891
|
+
list(branch.find_all(exp.Select)) if isinstance(branch, exp.Expr)
|
|
892
|
+
else [])
|
|
893
|
+
for sel in selects:
|
|
894
|
+
for e in sel.expressions:
|
|
895
|
+
node = e.this if isinstance(e, exp.Alias) else e
|
|
896
|
+
if isinstance(node, exp.Star):
|
|
897
|
+
return True
|
|
898
|
+
if isinstance(node, exp.Column) and isinstance(node.this, exp.Star):
|
|
899
|
+
return True # t.*
|
|
900
|
+
return False
|
|
901
|
+
|
|
902
|
+
|
|
903
|
+
def set_operation_column_position(stmt, loc, ctx):
|
|
904
|
+
out = []
|
|
905
|
+
for union in stmt.find_all(exp.Union, exp.Except, exp.Intersect):
|
|
906
|
+
branches = [union.this, union.args.get("expression")]
|
|
907
|
+
if any(isinstance(b, exp.Select) and b.args.get("from_") is None
|
|
908
|
+
for b in branches if b is not None):
|
|
909
|
+
continue # VALUES branches have no positional fragility
|
|
910
|
+
if any(b is not None and _branch_projection_star(b)
|
|
911
|
+
for b in branches):
|
|
912
|
+
out.append((union,
|
|
913
|
+
"set operation branches use SELECT *: positional mapping scrambles on reorder",
|
|
914
|
+
"list columns explicitly in the same order, or use BY NAME where supported"))
|
|
915
|
+
return out
|
|
916
|
+
|
|
917
|
+
|
|
918
|
+
# -- sql/average-of-ratios (heuristic, off by default) ----------------------------------
|
|
919
|
+
def average_of_ratios(stmt, loc, ctx):
|
|
920
|
+
out = []
|
|
921
|
+
for avg in stmt.find_all(exp.Avg):
|
|
922
|
+
arg = avg.this
|
|
923
|
+
if isinstance(arg, exp.Div):
|
|
924
|
+
out.append((avg,
|
|
925
|
+
"AVG(a/b) weights every group equally; usually SUM(a)/SUM(b) is meant",
|
|
926
|
+
"rewrite as SUM(num)/SUM(den), or weight explicitly"))
|
|
927
|
+
return out
|
|
928
|
+
|
|
929
|
+
|
|
930
|
+
# -- sql/between-date-on-timestamp --------------------------------------------------
|
|
931
|
+
_DATE_ONLY = re.compile(r"^\d{4}-\d{2}-\d{2}$")
|
|
932
|
+
# Column types are not in the file, so the column's name decides: these
|
|
933
|
+
# read as timestamps. A DATE column (order_date) is left alone, because
|
|
934
|
+
# BETWEEN over dates is correct.
|
|
935
|
+
_TIMESTAMP_NAME = re.compile(r"(_at|_ts|_time|time|timestamp|_dt)$", re.IGNORECASE)
|
|
936
|
+
|
|
937
|
+
|
|
938
|
+
def _date_only_bound(bound):
|
|
939
|
+
if isinstance(bound, exp.Cast) and bound.to.is_type("date"):
|
|
940
|
+
bound = bound.this # DATE '2025-03-31'
|
|
941
|
+
return isinstance(bound, exp.Literal) and bool(_DATE_ONLY.match(bound.name or ""))
|
|
942
|
+
|
|
943
|
+
|
|
944
|
+
def between_date_on_timestamp(stmt, loc, ctx):
|
|
945
|
+
out = []
|
|
946
|
+
for between in stmt.find_all(exp.Between):
|
|
947
|
+
col = between.this
|
|
948
|
+
if not (isinstance(col, exp.Column) and _TIMESTAMP_NAME.search(col.name)):
|
|
949
|
+
continue
|
|
950
|
+
if _date_only_bound(between.args.get("high")):
|
|
951
|
+
out.append((between,
|
|
952
|
+
"BETWEEN with a date-only upper bound on a timestamp "
|
|
953
|
+
"drops everything after midnight on the last day",
|
|
954
|
+
"use a half-open range: >= start AND < day-after-end"))
|
|
955
|
+
break
|
|
956
|
+
return out
|
|
957
|
+
|
|
958
|
+
|
|
959
|
+
# -- sql/distinct-dedup-bandaid (heuristic, off by default) ------------------------------
|
|
960
|
+
def distinct_dedup_bandaid(stmt, loc, ctx):
|
|
961
|
+
out = []
|
|
962
|
+
for sel in stmt.find_all(exp.Select):
|
|
963
|
+
if sel.args.get("distinct") is None:
|
|
964
|
+
continue
|
|
965
|
+
agg_inside = sel.find(exp.Sum, exp.Count, exp.Avg) is not None
|
|
966
|
+
agg_outside = False
|
|
967
|
+
if not agg_inside:
|
|
968
|
+
# DISTINCT in a derived table feeding an outer aggregate:
|
|
969
|
+
# SELECT SUM(x) FROM (SELECT DISTINCT ...)
|
|
970
|
+
node = sel.parent
|
|
971
|
+
while node is not None and not isinstance(node, exp.Select):
|
|
972
|
+
node = getattr(node, "parent", None)
|
|
973
|
+
if isinstance(node, exp.Select) and node is not sel:
|
|
974
|
+
agg_outside = node.find(exp.Sum, exp.Count, exp.Avg) is not None
|
|
975
|
+
if agg_inside or agg_outside:
|
|
976
|
+
out.append((sel,
|
|
977
|
+
"SELECT DISTINCT over aggregates usually papers over a join fanout",
|
|
978
|
+
"fix the fanout (pre-aggregate before joining) instead of deduplicating"))
|
|
979
|
+
break
|
|
980
|
+
return out
|
|
981
|
+
|
|
982
|
+
|
|
983
|
+
# -- sql/insert-explicit-column-list --------------------------------------------------
|
|
984
|
+
def insert_explicit_column_list(stmt, loc, ctx):
|
|
985
|
+
out = []
|
|
986
|
+
for insert in stmt.find_all(exp.Insert):
|
|
987
|
+
if isinstance(insert.this, exp.Table):
|
|
988
|
+
out.append((insert,
|
|
989
|
+
"INSERT without a column list breaks silently when columns are added or reordered",
|
|
990
|
+
"name the columns: INSERT INTO t (a, b) SELECT ..."))
|
|
991
|
+
return out
|
|
992
|
+
|
|
993
|
+
|
|
994
|
+
# -- sql/integer-division-truncation --------------------------------------------------
|
|
995
|
+
def _is_int_literal(node):
|
|
996
|
+
return isinstance(node, exp.Literal) and re.fullmatch(r"\d+", node.name or "")
|
|
997
|
+
|
|
998
|
+
|
|
999
|
+
# Engines where `/` between two integers returns a non-integer result.
|
|
1000
|
+
# Everywhere else (Postgres, SQL Server, SQLite, Redshift, Trino, and an
|
|
1001
|
+
# unknown dialect) `1 / 2` truncates to 0.
|
|
1002
|
+
_TRUE_DIVISION_DIALECTS = frozenset({
|
|
1003
|
+
"bigquery", "clickhouse", "databricks", "doris", "duckdb", "hive",
|
|
1004
|
+
"mysql", "oracle", "snowflake", "spark", "spark2", "starrocks",
|
|
1005
|
+
})
|
|
1006
|
+
|
|
1007
|
+
|
|
1008
|
+
def integer_division_truncation(stmt, loc, ctx):
|
|
1009
|
+
"""Known recall limit: division between *columns* (count(x)/count(*))
|
|
1010
|
+
is not flagged — column types are unknowable statically. Only
|
|
1011
|
+
integer literals are caught: `/` where the dialect truncates, and the
|
|
1012
|
+
explicit integer-division operator `//` / DIV everywhere. The receipt
|
|
1013
|
+
lists column-typed division under 'not checked'."""
|
|
1014
|
+
out = []
|
|
1015
|
+
true_division = (ctx.dialect or "").lower() in _TRUE_DIVISION_DIALECTS
|
|
1016
|
+
for div in stmt.find_all(exp.Div, exp.IntDiv):
|
|
1017
|
+
if not (_is_int_literal(div.left) and _is_int_literal(div.right)):
|
|
1018
|
+
continue
|
|
1019
|
+
if isinstance(div, exp.IntDiv):
|
|
1020
|
+
out.append((div, "integer division truncates (5 // 2 = 2)",
|
|
1021
|
+
"use / with a decimal operand if a fraction is meant"))
|
|
1022
|
+
elif not true_division:
|
|
1023
|
+
out.append((div,
|
|
1024
|
+
"integer/integer division truncates (1 / 2 = 0) in "
|
|
1025
|
+
"Postgres, SQL Server, SQLite, Redshift and Trino",
|
|
1026
|
+
"multiply by 1.0 or cast one side to a float/decimal"))
|
|
1027
|
+
return out
|
|
1028
|
+
|
|
1029
|
+
|
|
1030
|
+
# -- sql/keyset-pagination (heuristic, off by default) ----------------------------------
|
|
1031
|
+
def keyset_pagination(stmt, loc, ctx):
|
|
1032
|
+
out = []
|
|
1033
|
+
for sel in stmt.find_all(exp.Select):
|
|
1034
|
+
if sel.args.get("offset") is not None and sel.args.get("limit") is not None:
|
|
1035
|
+
out.append((sel,
|
|
1036
|
+
"LIMIT+OFFSET pages shift under concurrent writes",
|
|
1037
|
+
"paginate by keyset: WHERE id > ? ORDER BY id LIMIT ?"))
|
|
1038
|
+
return out
|
|
1039
|
+
|
|
1040
|
+
|
|
1041
|
+
# -- sql/limit-requires-order-by (heuristic, off by default) ------------------------------
|
|
1042
|
+
def limit_requires_order_by(stmt, loc, ctx):
|
|
1043
|
+
out = []
|
|
1044
|
+
for sel in stmt.find_all(exp.Select):
|
|
1045
|
+
if sel.args.get("limit") is not None and sel.args.get("order") is None:
|
|
1046
|
+
# subqueries feeding an ordered outer query are fine
|
|
1047
|
+
parent = sel.parent
|
|
1048
|
+
if isinstance(parent, exp.Subquery) and isinstance(parent.parent, exp.Select):
|
|
1049
|
+
continue
|
|
1050
|
+
out.append((sel,
|
|
1051
|
+
"LIMIT without ORDER BY returns an arbitrary, unstable row set",
|
|
1052
|
+
"add ORDER BY a unique key"))
|
|
1053
|
+
return out
|
|
1054
|
+
|
|
1055
|
+
|
|
1056
|
+
# -- sql/natural-join-explicit-keys --------------------------------------------------
|
|
1057
|
+
def natural_join_explicit_keys(stmt, loc, ctx):
|
|
1058
|
+
out = []
|
|
1059
|
+
for join in stmt.find_all(exp.Join):
|
|
1060
|
+
if (join.args.get("method") or "").upper() == "NATURAL":
|
|
1061
|
+
out.append((join,
|
|
1062
|
+
"NATURAL JOIN matches on every same-named column, including future ones",
|
|
1063
|
+
"name the keys: JOIN ... ON / USING"))
|
|
1064
|
+
return out
|
|
1065
|
+
|
|
1066
|
+
|
|
1067
|
+
# -- sql/negative-filter-drops-nulls --------------------------------------------------
|
|
1068
|
+
def _has_tolerant_wrapper(pred):
|
|
1069
|
+
return any(isinstance(f, exp.Func) and f.sql_name().lower() in TOLERANT_FNS
|
|
1070
|
+
for f in pred.find_all(exp.Func))
|
|
1071
|
+
|
|
1072
|
+
|
|
1073
|
+
def negative_filter_drops_nulls(stmt, loc, ctx):
|
|
1074
|
+
out = []
|
|
1075
|
+
for pred in list(stmt.find_all(exp.NEQ)) + list(stmt.find_all(exp.Not)):
|
|
1076
|
+
if isinstance(pred, exp.NEQ) and any(
|
|
1077
|
+
isinstance(x, exp.Null) for x in (pred.left, pred.right)):
|
|
1078
|
+
continue # `x <> NULL` belongs to null-equality-comparison
|
|
1079
|
+
if _has_tolerant_wrapper(pred):
|
|
1080
|
+
continue # COALESCE-class wrappers keep NULLs explicitly
|
|
1081
|
+
if isinstance(pred, exp.Not):
|
|
1082
|
+
inner = pred.this
|
|
1083
|
+
while isinstance(inner, exp.Paren):
|
|
1084
|
+
inner = inner.this
|
|
1085
|
+
if not isinstance(inner, (exp.Like, exp.ILike, exp.In)):
|
|
1086
|
+
continue
|
|
1087
|
+
if isinstance(inner, exp.In) and (
|
|
1088
|
+
inner.find(exp.Subquery) or inner.find(exp.Select)):
|
|
1089
|
+
continue # owned by sql/not-in-nullable-subquery
|
|
1090
|
+
if _or_null_sibling(pred):
|
|
1091
|
+
continue # `... OR col IS NULL` keeps NULLs explicitly
|
|
1092
|
+
out.append((pred,
|
|
1093
|
+
"negative filters (<>, NOT LIKE, NOT IN-list) silently drop NULL rows",
|
|
1094
|
+
"add OR col IS NULL when NULLs should be kept"))
|
|
1095
|
+
return out
|
|
1096
|
+
|
|
1097
|
+
|
|
1098
|
+
def _or_null_sibling(pred):
|
|
1099
|
+
node = pred.parent
|
|
1100
|
+
while node is not None and not isinstance(node, (exp.Where, exp.Select)):
|
|
1101
|
+
if isinstance(node, exp.Or):
|
|
1102
|
+
return node.find(exp.Is) is not None
|
|
1103
|
+
node = getattr(node, "parent", None)
|
|
1104
|
+
return False
|
|
1105
|
+
|
|
1106
|
+
|
|
1107
|
+
# -- sql/row-number-dedupe-tiebreaker --------------------------------------------------
|
|
1108
|
+
def _picks_one_row(stmt, window):
|
|
1109
|
+
"""True when the ROW_NUMBER result is filtered to one row per group:
|
|
1110
|
+
QUALIFY rn = 1, or an outer WHERE rn = 1 over this window's alias."""
|
|
1111
|
+
node = window
|
|
1112
|
+
while node is not None:
|
|
1113
|
+
if isinstance(node, exp.Qualify):
|
|
1114
|
+
return True
|
|
1115
|
+
if isinstance(node, exp.Select):
|
|
1116
|
+
qualify = node.args.get("qualify")
|
|
1117
|
+
if qualify is not None and _qualify_filters(qualify):
|
|
1118
|
+
return True
|
|
1119
|
+
if node is stmt:
|
|
1120
|
+
break
|
|
1121
|
+
node = getattr(node, "parent", None)
|
|
1122
|
+
alias = None
|
|
1123
|
+
node = window.parent
|
|
1124
|
+
while node is not None and not isinstance(node, exp.Select):
|
|
1125
|
+
if isinstance(node, exp.Alias):
|
|
1126
|
+
alias = node.args.get("alias")
|
|
1127
|
+
break
|
|
1128
|
+
node = getattr(node, "parent", None)
|
|
1129
|
+
if alias is None:
|
|
1130
|
+
return False
|
|
1131
|
+
name = alias.name.lower() if hasattr(alias, "name") else str(alias).lower()
|
|
1132
|
+
for sel in stmt.find_all(exp.Select):
|
|
1133
|
+
where = sel.args.get("where")
|
|
1134
|
+
if where is None:
|
|
1135
|
+
continue
|
|
1136
|
+
for eq in where.find_all(exp.EQ):
|
|
1137
|
+
for side in (eq.left, eq.right):
|
|
1138
|
+
if isinstance(side, exp.Column) and side.name.lower() == name:
|
|
1139
|
+
other = eq.right if side is eq.left else eq.left
|
|
1140
|
+
if isinstance(other, exp.Literal) and other.name == "1":
|
|
1141
|
+
return True
|
|
1142
|
+
return False
|
|
1143
|
+
|
|
1144
|
+
|
|
1145
|
+
def _qualify_filters(qualify):
|
|
1146
|
+
return any(isinstance(node, (exp.EQ, exp.LT, exp.LTE))
|
|
1147
|
+
for node in qualify.find_all(exp.EQ, exp.LT, exp.LTE))
|
|
1148
|
+
|
|
1149
|
+
|
|
1150
|
+
def row_number_dedupe_tiebreaker(stmt, loc, ctx):
|
|
1151
|
+
out = []
|
|
1152
|
+
for window in stmt.find_all(exp.Window):
|
|
1153
|
+
if window.find(exp.RowNumber) is None:
|
|
1154
|
+
continue
|
|
1155
|
+
order = window.args.get("order")
|
|
1156
|
+
if order is None:
|
|
1157
|
+
if _picks_one_row(stmt, window):
|
|
1158
|
+
out.append((window,
|
|
1159
|
+
"ROW_NUMBER without ORDER BY keeps an arbitrary row "
|
|
1160
|
+
"per group, and a different one on each run",
|
|
1161
|
+
"order by the column that picks the winner, then "
|
|
1162
|
+
"a unique tiebreaker"))
|
|
1163
|
+
continue
|
|
1164
|
+
keys = order.expressions if hasattr(order, "expressions") else []
|
|
1165
|
+
if len(keys) == 1 and _picks_one_row(stmt, window):
|
|
1166
|
+
out.append((window,
|
|
1167
|
+
"ROW_NUMBER with a single ORDER BY key picks an arbitrary "
|
|
1168
|
+
"winner on ties (uniqueness can't be verified from the file)",
|
|
1169
|
+
"add a unique tiebreaker column as the last ORDER BY key"))
|
|
1170
|
+
return out
|
|
1171
|
+
|
|
1172
|
+
|
|
1173
|
+
# -- sql/union-all-default (heuristic, off by default) ----------------------------------
|
|
1174
|
+
def union_all_default(stmt, loc, ctx):
|
|
1175
|
+
out = []
|
|
1176
|
+
for union in stmt.find_all(exp.Union):
|
|
1177
|
+
if union.args.get("distinct") is True:
|
|
1178
|
+
out.append((union,
|
|
1179
|
+
"UNION deduplicates; most pipelines mean UNION ALL",
|
|
1180
|
+
"use UNION ALL unless dedup is the intent (and affordable)"))
|
|
1181
|
+
return out
|
|
1182
|
+
|
|
1183
|
+
|
|
1184
|
+
# -- sql/group-by-bare-column (permissive dialects only) ----------------------------------
|
|
1185
|
+
_PERMISSIVE_GROUP_BY = ("mysql", "sqlite")
|
|
1186
|
+
|
|
1187
|
+
|
|
1188
|
+
def group_by_bare_column(stmt, loc, ctx):
|
|
1189
|
+
if (ctx.dialect or "generic") not in _PERMISSIVE_GROUP_BY:
|
|
1190
|
+
return [] # DuckDB/Postgres raise; nothing to flag
|
|
1191
|
+
out = []
|
|
1192
|
+
for sel in stmt.find_all(exp.Select):
|
|
1193
|
+
group = sel.args.get("group")
|
|
1194
|
+
if group is None:
|
|
1195
|
+
continue
|
|
1196
|
+
if any(isinstance(e, exp.Literal) for e in group.expressions):
|
|
1197
|
+
continue # GROUP BY 1, 2: positions, unknowable without resolving
|
|
1198
|
+
keys = {e.sql().lower() for e in group.expressions}
|
|
1199
|
+
keys |= {e.name.lower() for e in group.expressions
|
|
1200
|
+
if isinstance(e, exp.Column)}
|
|
1201
|
+
for proj in sel.expressions:
|
|
1202
|
+
col = proj.unalias() if hasattr(proj, "unalias") else proj
|
|
1203
|
+
grouped = {col.sql().lower(), col.name.lower(),
|
|
1204
|
+
(proj.alias or "").lower()} & keys
|
|
1205
|
+
if isinstance(col, exp.Column) and not grouped:
|
|
1206
|
+
if sel.find(exp.Sum, exp.Count, exp.Avg, exp.Min, exp.Max):
|
|
1207
|
+
out.append((proj,
|
|
1208
|
+
"bare column not in GROUP BY returns an arbitrary value (MySQL/SQLite)",
|
|
1209
|
+
"add it to GROUP BY or aggregate it"))
|
|
1210
|
+
break
|
|
1211
|
+
return out
|
|
1212
|
+
|
|
1213
|
+
|
|
1214
|
+
# -- sql/case-without-else-in-aggregate --------------------------------------------------
|
|
1215
|
+
_AGGS = (exp.Sum, exp.Avg) # record scope: SUM, AVG. COUNT is the
|
|
1216
|
+
# conditional-count idiom; MAX/MIN(CASE) without ELSE is the conditional
|
|
1217
|
+
# extremum idiom (NULL when nothing matches is the wanted answer).
|
|
1218
|
+
|
|
1219
|
+
|
|
1220
|
+
def _is_null_guarded(agg):
|
|
1221
|
+
"""True when the aggregate sits inside a COALESCE/IFNULL/NVL call in
|
|
1222
|
+
the same query level (sqlglot normalizes all three to Coalesce).
|
|
1223
|
+
The guard supplies the neutral value for the all-NULL case, so the
|
|
1224
|
+
missing ELSE is the intended conditional-aggregate idiom, not a
|
|
1225
|
+
silent skip."""
|
|
1226
|
+
node = getattr(agg, "parent", None)
|
|
1227
|
+
while node is not None:
|
|
1228
|
+
if isinstance(node, exp.Select):
|
|
1229
|
+
return False
|
|
1230
|
+
if isinstance(node, exp.Coalesce):
|
|
1231
|
+
return True
|
|
1232
|
+
node = getattr(node, "parent", None)
|
|
1233
|
+
return False
|
|
1234
|
+
|
|
1235
|
+
|
|
1236
|
+
def case_without_else_in_aggregate(stmt, loc, ctx):
|
|
1237
|
+
out = []
|
|
1238
|
+
for agg in stmt.find_all(*_AGGS):
|
|
1239
|
+
if _is_null_guarded(agg):
|
|
1240
|
+
continue
|
|
1241
|
+
for case in agg.find_all(exp.Case):
|
|
1242
|
+
if case.args.get("default") is None:
|
|
1243
|
+
# ELSE 0 is right for SUM and wrong for AVG (it drags the
|
|
1244
|
+
# mean toward zero); the fix names the neutral value.
|
|
1245
|
+
fix = ("write ELSE NULL to make the skip explicit (ELSE 0 "
|
|
1246
|
+
"would change the mean)"
|
|
1247
|
+
if isinstance(agg, exp.Avg) else
|
|
1248
|
+
"add ELSE 0 (or ELSE <neutral>) inside aggregates")
|
|
1249
|
+
out.append((case,
|
|
1250
|
+
"CASE without ELSE yields NULL, which aggregates silently skip",
|
|
1251
|
+
fix))
|
|
1252
|
+
break
|
|
1253
|
+
return out
|
|
1254
|
+
|
|
1255
|
+
|
|
1256
|
+
CHECKS = {
|
|
1257
|
+
"sql/not-in-nullable-subquery": not_in_nullable_subquery,
|
|
1258
|
+
"sql/unguarded-update-delete": unguarded_update_delete,
|
|
1259
|
+
"sql/accidental-cross-join": accidental_cross_join,
|
|
1260
|
+
"sql/left-join-null-rejecting-where": left_join_null_rejecting_where,
|
|
1261
|
+
"sql/null-equality-comparison": null_equality_comparison,
|
|
1262
|
+
"sql/count-star-after-left-join": count_star_after_left_join,
|
|
1263
|
+
"sql/nondeterministic-merge-keys": nondeterministic_merge_keys,
|
|
1264
|
+
"sql/set-operation-column-position": set_operation_column_position,
|
|
1265
|
+
"sql/average-of-ratios": average_of_ratios,
|
|
1266
|
+
"sql/between-date-on-timestamp": between_date_on_timestamp,
|
|
1267
|
+
"sql/distinct-dedup-bandaid": distinct_dedup_bandaid,
|
|
1268
|
+
"sql/insert-explicit-column-list": insert_explicit_column_list,
|
|
1269
|
+
"sql/integer-division-truncation": integer_division_truncation,
|
|
1270
|
+
"sql/keyset-pagination": keyset_pagination,
|
|
1271
|
+
"sql/limit-requires-order-by": limit_requires_order_by,
|
|
1272
|
+
"sql/natural-join-explicit-keys": natural_join_explicit_keys,
|
|
1273
|
+
"sql/negative-filter-drops-nulls": negative_filter_drops_nulls,
|
|
1274
|
+
"sql/row-number-dedupe-tiebreaker": row_number_dedupe_tiebreaker,
|
|
1275
|
+
"sql/union-all-default": union_all_default,
|
|
1276
|
+
"sql/group-by-bare-column": group_by_bare_column,
|
|
1277
|
+
"sql/case-without-else-in-aggregate": case_without_else_in_aggregate,
|
|
1278
|
+
}
|