dataform-sqlx-lint 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dataform_sqlx_lint/__init__.py +7 -0
- dataform_sqlx_lint/cli.py +81 -0
- dataform_sqlx_lint/config.py +110 -0
- dataform_sqlx_lint/linter.py +378 -0
- dataform_sqlx_lint-0.1.0.dist-info/METADATA +124 -0
- dataform_sqlx_lint-0.1.0.dist-info/RECORD +10 -0
- dataform_sqlx_lint-0.1.0.dist-info/WHEEL +5 -0
- dataform_sqlx_lint-0.1.0.dist-info/entry_points.txt +2 -0
- dataform_sqlx_lint-0.1.0.dist-info/licenses/LICENSE +21 -0
- dataform_sqlx_lint-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""dataform-sqlx-lint: convention linter for Dataform .sqlx files."""
|
|
2
|
+
|
|
3
|
+
from .config import Config, DirPolicy, load_config
|
|
4
|
+
from .linter import Finding, lint_file, lint_text
|
|
5
|
+
|
|
6
|
+
__all__ = ["Config", "DirPolicy", "Finding", "lint_file", "lint_text", "load_config"]
|
|
7
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""Command-line interface.
|
|
2
|
+
|
|
3
|
+
Usage: dataform-sqlx-lint [--config PATH] [--definitions-root DIR] FILE [FILE ...]
|
|
4
|
+
Exit codes: 0 = clean or warnings only, 1 = errors found, 2 = usage error.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import argparse
|
|
10
|
+
import sys
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
import tomllib
|
|
14
|
+
|
|
15
|
+
from .config import Config, _from_dict, load_config
|
|
16
|
+
from .linter import lint_file
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _repo_resolver(root: Path):
|
|
20
|
+
"""Resolve an action name to the text of <root>/**/<name>.sqlx.
|
|
21
|
+
Used to follow `select *` through ${ref()} for E010."""
|
|
22
|
+
if not root.is_dir():
|
|
23
|
+
return lambda name: None
|
|
24
|
+
index = {p.stem: p for p in root.rglob("*.sqlx")}
|
|
25
|
+
|
|
26
|
+
def resolve(name):
|
|
27
|
+
p = index.get(name)
|
|
28
|
+
try:
|
|
29
|
+
return p.read_text(encoding="utf-8") if p else None
|
|
30
|
+
except OSError:
|
|
31
|
+
return None
|
|
32
|
+
|
|
33
|
+
return resolve
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def main(argv: list[str] | None = None) -> int:
|
|
37
|
+
parser = argparse.ArgumentParser(
|
|
38
|
+
prog="dataform-sqlx-lint",
|
|
39
|
+
description="Convention linter for Dataform .sqlx files",
|
|
40
|
+
)
|
|
41
|
+
parser.add_argument("files", nargs="+", help=".sqlx files to lint")
|
|
42
|
+
parser.add_argument(
|
|
43
|
+
"--config",
|
|
44
|
+
help="path to a TOML config file (default: .sqlx-lint.toml or "
|
|
45
|
+
"[tool.sqlx-lint] in ./pyproject.toml)",
|
|
46
|
+
)
|
|
47
|
+
parser.add_argument(
|
|
48
|
+
"--definitions-root",
|
|
49
|
+
default="definitions",
|
|
50
|
+
help="directory indexed to resolve ${ref()} targets for the E010 "
|
|
51
|
+
"coverage rule (default: ./definitions)",
|
|
52
|
+
)
|
|
53
|
+
args = parser.parse_args(argv)
|
|
54
|
+
|
|
55
|
+
if args.config:
|
|
56
|
+
cfg: Config = _from_dict(tomllib.loads(Path(args.config).read_text()))
|
|
57
|
+
else:
|
|
58
|
+
cfg = load_config(".")
|
|
59
|
+
resolver = _repo_resolver(Path(args.definitions_root))
|
|
60
|
+
|
|
61
|
+
errors = warnings = 0
|
|
62
|
+
for path in args.files:
|
|
63
|
+
try:
|
|
64
|
+
findings = lint_file(path, config=cfg, resolver=resolver)
|
|
65
|
+
except OSError as exc:
|
|
66
|
+
print(f"{path}: cannot read: {exc}")
|
|
67
|
+
errors += 1
|
|
68
|
+
continue
|
|
69
|
+
for f in sorted(findings, key=lambda f: f.line):
|
|
70
|
+
print(f"{path}:{f.line}: {f.code} [{f.severity}] {f.message}")
|
|
71
|
+
if f.severity == "error":
|
|
72
|
+
errors += 1
|
|
73
|
+
else:
|
|
74
|
+
warnings += 1
|
|
75
|
+
if errors or warnings:
|
|
76
|
+
print(f"sqlx-lint: {errors} error(s), {warnings} warning(s)")
|
|
77
|
+
return 1 if errors else 0
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def entrypoint() -> None:
|
|
81
|
+
sys.exit(main())
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"""Configuration model and TOML loading.
|
|
2
|
+
|
|
3
|
+
Precedence: .sqlx-lint.toml in the working directory, else the
|
|
4
|
+
[tool.sqlx-lint] table of pyproject.toml, else built-in defaults.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import tomllib
|
|
10
|
+
from dataclasses import dataclass, field
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
#: Rules that run unless disabled.
|
|
14
|
+
DEFAULT_ENABLED = {"E001", "E002", "E003", "E004", "E006", "E007", "E010"}
|
|
15
|
+
#: Opt-in rules (house-style checks): enable via `enable = [...]`.
|
|
16
|
+
OPT_IN = {"E005", "W008"}
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass(frozen=True)
|
|
20
|
+
class DirPolicy:
|
|
21
|
+
"""Naming/type policy applied to files whose path contains a substring."""
|
|
22
|
+
|
|
23
|
+
path_contains: str
|
|
24
|
+
require_prefix: str | None = None
|
|
25
|
+
require_types: tuple[str, ...] | None = None
|
|
26
|
+
severity: str = "error"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class Config:
|
|
31
|
+
schema_suffixes: list[str] = field(default_factory=lambda: ["_prod", "_dev"])
|
|
32
|
+
documented_types: set[str] = field(
|
|
33
|
+
default_factory=lambda: {"table", "view", "incremental", "declaration"}
|
|
34
|
+
)
|
|
35
|
+
#: E010 applies only to files whose path contains one of these; empty = all.
|
|
36
|
+
coverage_paths: list[str] = field(default_factory=list)
|
|
37
|
+
dir_policies: list[DirPolicy] = field(default_factory=list)
|
|
38
|
+
enabled_extra: set[str] = field(default_factory=set)
|
|
39
|
+
disabled: set[str] = field(default_factory=set)
|
|
40
|
+
|
|
41
|
+
def rule_on(self, code: str) -> bool:
|
|
42
|
+
if code in self.disabled:
|
|
43
|
+
return False
|
|
44
|
+
return code in DEFAULT_ENABLED or code in self.enabled_extra
|
|
45
|
+
|
|
46
|
+
def __eq__(self, other):
|
|
47
|
+
if not isinstance(other, Config):
|
|
48
|
+
return NotImplemented
|
|
49
|
+
return (
|
|
50
|
+
self.schema_suffixes == other.schema_suffixes
|
|
51
|
+
and self.documented_types == other.documented_types
|
|
52
|
+
and self.coverage_paths == other.coverage_paths
|
|
53
|
+
and self.dir_policies == other.dir_policies
|
|
54
|
+
and self.enabled_extra == other.enabled_extra
|
|
55
|
+
and self.disabled == other.disabled
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
_KNOWN_KEYS = {
|
|
60
|
+
"schema_suffixes",
|
|
61
|
+
"documented_types",
|
|
62
|
+
"coverage_paths",
|
|
63
|
+
"dir_policies",
|
|
64
|
+
"enable",
|
|
65
|
+
"disable",
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _from_dict(raw: dict) -> Config:
|
|
70
|
+
unknown = set(raw) - _KNOWN_KEYS
|
|
71
|
+
if unknown:
|
|
72
|
+
raise ValueError(
|
|
73
|
+
f"unknown sqlx-lint config key(s): {', '.join(sorted(unknown))}"
|
|
74
|
+
)
|
|
75
|
+
policies = [
|
|
76
|
+
DirPolicy(
|
|
77
|
+
path_contains=p["path_contains"],
|
|
78
|
+
require_prefix=p.get("require_prefix"),
|
|
79
|
+
require_types=tuple(p["require_types"]) if p.get("require_types") else None,
|
|
80
|
+
severity=p.get("severity", "error"),
|
|
81
|
+
)
|
|
82
|
+
for p in raw.get("dir_policies", [])
|
|
83
|
+
]
|
|
84
|
+
kwargs = {}
|
|
85
|
+
if "schema_suffixes" in raw:
|
|
86
|
+
kwargs["schema_suffixes"] = list(raw["schema_suffixes"])
|
|
87
|
+
if "documented_types" in raw:
|
|
88
|
+
kwargs["documented_types"] = set(raw["documented_types"])
|
|
89
|
+
if "coverage_paths" in raw:
|
|
90
|
+
kwargs["coverage_paths"] = list(raw["coverage_paths"])
|
|
91
|
+
return Config(
|
|
92
|
+
dir_policies=policies,
|
|
93
|
+
enabled_extra=set(raw.get("enable", [])),
|
|
94
|
+
disabled=set(raw.get("disable", [])),
|
|
95
|
+
**kwargs,
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def load_config(root: str | Path = ".") -> Config:
|
|
100
|
+
root = Path(root)
|
|
101
|
+
standalone = root / ".sqlx-lint.toml"
|
|
102
|
+
if standalone.is_file():
|
|
103
|
+
return _from_dict(tomllib.loads(standalone.read_text(encoding="utf-8")))
|
|
104
|
+
pyproject = root / "pyproject.toml"
|
|
105
|
+
if pyproject.is_file():
|
|
106
|
+
data = tomllib.loads(pyproject.read_text(encoding="utf-8"))
|
|
107
|
+
tool = data.get("tool", {}).get("sqlx-lint")
|
|
108
|
+
if tool is not None:
|
|
109
|
+
return _from_dict(tool)
|
|
110
|
+
return Config()
|
|
@@ -0,0 +1,378 @@
|
|
|
1
|
+
"""Core linter: parses the .sqlx config block and SQL body, emits Findings.
|
|
2
|
+
|
|
3
|
+
Rules (codes are stable; suppress with `-- sqlx-lint: disable=E006` on the
|
|
4
|
+
offending line or `-- sqlx-lint: disable-file=E006` anywhere in the file):
|
|
5
|
+
|
|
6
|
+
E001 config {} block missing or unbalanced
|
|
7
|
+
E002 columns: {} missing or empty on documented types
|
|
8
|
+
E003 schema: hardcodes an environment suffix (suffix-doubling trap)
|
|
9
|
+
E004 name: redundantly matches the filename (non-declarations)
|
|
10
|
+
E005 schema: set on operations/assertion configs [opt-in]
|
|
11
|
+
E006 hardcoded `project.dataset.table` path instead of ${ref()}
|
|
12
|
+
E007 directory policy violation (configured prefix/type per path)
|
|
13
|
+
W008 post_operations block appears before the main SELECT [opt-in]
|
|
14
|
+
E010 columns:{} does not cover every determinable output column
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import re
|
|
20
|
+
from dataclasses import dataclass
|
|
21
|
+
|
|
22
|
+
from .config import Config
|
|
23
|
+
|
|
24
|
+
NO_SCHEMA_TYPES = {"operations", "assertion", "test"}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass
|
|
28
|
+
class Finding:
|
|
29
|
+
code: str
|
|
30
|
+
severity: str # "error" | "warning"
|
|
31
|
+
line: int
|
|
32
|
+
message: str
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _line_of(text, pos):
|
|
36
|
+
return text.count("\n", 0, pos) + 1
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _balanced_block(text, open_brace_pos):
|
|
40
|
+
"""Return (content, end_pos) of the brace block at open_brace_pos, else None."""
|
|
41
|
+
depth = 0
|
|
42
|
+
in_str = None
|
|
43
|
+
i = open_brace_pos
|
|
44
|
+
while i < len(text):
|
|
45
|
+
ch = text[i]
|
|
46
|
+
if in_str:
|
|
47
|
+
if ch == "\\":
|
|
48
|
+
i += 2
|
|
49
|
+
continue
|
|
50
|
+
if ch == in_str:
|
|
51
|
+
in_str = None
|
|
52
|
+
elif ch in "\"'`":
|
|
53
|
+
in_str = ch
|
|
54
|
+
elif ch == "{":
|
|
55
|
+
depth += 1
|
|
56
|
+
elif ch == "}":
|
|
57
|
+
depth -= 1
|
|
58
|
+
if depth == 0:
|
|
59
|
+
return text[open_brace_pos + 1 : i], i + 1
|
|
60
|
+
i += 1
|
|
61
|
+
return None
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _config_value(config, key):
|
|
65
|
+
m = re.search(rf'\b{key}\s*:\s*"([^"]*)"', config)
|
|
66
|
+
return m.group(1) if m else None
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _strip_comments(text):
|
|
70
|
+
text = re.sub(
|
|
71
|
+
r"/\*.*?\*/", lambda m: re.sub(r"[^\n]", " ", m.group(0)), text, flags=re.S
|
|
72
|
+
)
|
|
73
|
+
return re.sub(r"--[^\n]*", lambda m: " " * len(m.group(0)), text)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _split_top_level(text, sep=","):
|
|
77
|
+
"""Split text on sep occurring outside (), [], {}, and string literals."""
|
|
78
|
+
parts, depth, start, in_str, i = [], 0, 0, None, 0
|
|
79
|
+
while i < len(text):
|
|
80
|
+
ch = text[i]
|
|
81
|
+
if in_str:
|
|
82
|
+
if ch == "\\":
|
|
83
|
+
i += 2
|
|
84
|
+
continue
|
|
85
|
+
if ch == in_str:
|
|
86
|
+
in_str = None
|
|
87
|
+
elif ch in "\"'`":
|
|
88
|
+
in_str = ch
|
|
89
|
+
elif ch in "([{":
|
|
90
|
+
depth += 1
|
|
91
|
+
elif ch in ")]}":
|
|
92
|
+
depth -= 1
|
|
93
|
+
elif ch == sep and depth == 0:
|
|
94
|
+
parts.append(text[start:i])
|
|
95
|
+
start = i + 1
|
|
96
|
+
i += 1
|
|
97
|
+
parts.append(text[start:])
|
|
98
|
+
return parts
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _main_projection(clean_body):
|
|
102
|
+
"""Return (projection_text, from_clause_text, select_pos) of the first
|
|
103
|
+
top-level SELECT (CTE bodies and sub-selects sit inside parens/braces),
|
|
104
|
+
or None if no top-level SELECT exists."""
|
|
105
|
+
depth, in_str, i = 0, None, 0
|
|
106
|
+
select_pos = None
|
|
107
|
+
while i < len(clean_body):
|
|
108
|
+
ch = clean_body[i]
|
|
109
|
+
if in_str:
|
|
110
|
+
if ch == "\\":
|
|
111
|
+
i += 2
|
|
112
|
+
continue
|
|
113
|
+
if ch == in_str:
|
|
114
|
+
in_str = None
|
|
115
|
+
elif ch in "\"'`":
|
|
116
|
+
in_str = ch
|
|
117
|
+
elif ch in "([{":
|
|
118
|
+
depth += 1
|
|
119
|
+
elif ch in ")]}":
|
|
120
|
+
depth -= 1
|
|
121
|
+
elif depth == 0 and ch in "sSfF":
|
|
122
|
+
word = re.match(r"(select|from)\b", clean_body[i:], re.I)
|
|
123
|
+
if word and (
|
|
124
|
+
i == 0 or not (clean_body[i - 1].isalnum() or clean_body[i - 1] in "_$")
|
|
125
|
+
):
|
|
126
|
+
if word.group(1).lower() == "select" and select_pos is None:
|
|
127
|
+
select_pos = i
|
|
128
|
+
elif word.group(1).lower() == "from" and select_pos is not None:
|
|
129
|
+
return (clean_body[select_pos + 6 : i], clean_body[i:], select_pos)
|
|
130
|
+
i += 1
|
|
131
|
+
if select_pos is not None: # SELECT without FROM (constants)
|
|
132
|
+
return (clean_body[select_pos + 6 :], "", select_pos)
|
|
133
|
+
return None
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
_SIMPLE_IDENT = re.compile(r"^`?[A-Za-z_]\w*`?(?:\.`?([A-Za-z_]\w*)`?)?$")
|
|
137
|
+
_TRAILING_ALIAS = re.compile(r"\bas\s+`?([A-Za-z_]\w*)`?\s*$", re.I | re.S)
|
|
138
|
+
_STAR_ITEM = re.compile(
|
|
139
|
+
r"^(?:[A-Za-z_]\w*\.)?\*\s*(?:except\s*\(([^)]*)\))?\s*(?:replace\s*\(.*\))?$",
|
|
140
|
+
re.I | re.S,
|
|
141
|
+
)
|
|
142
|
+
_SINGLE_REF_FROM = re.compile(
|
|
143
|
+
r"^from\s+\$\{\s*ref\(\s*(?:\"[^\"]*\"\s*,\s*)?\"([^\"]+)\"\s*\)\s*\}\s*"
|
|
144
|
+
r"(?:as\s+\w+\s*)?"
|
|
145
|
+
r"(?:where\b|group\b|order\b|qualify\b|limit\b|window\b|post_operations\b|$)",
|
|
146
|
+
re.I | re.S,
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _known_output_columns(clean_body, resolver, _depth=0, _seen=None):
|
|
151
|
+
"""Best-effort set of output column names of a file's main SELECT.
|
|
152
|
+
Unparseable items are silently skipped; `select *` is followed through a
|
|
153
|
+
single plain ${ref()} when a resolver is provided. Never raises."""
|
|
154
|
+
parsed = _main_projection(clean_body)
|
|
155
|
+
if parsed is None:
|
|
156
|
+
return set()
|
|
157
|
+
projection, from_clause, _ = parsed
|
|
158
|
+
names, star_except = set(), None
|
|
159
|
+
items = _split_top_level(projection)
|
|
160
|
+
if items:
|
|
161
|
+
items[0] = re.sub(r"^\s*(distinct|all)\b", "", items[0], flags=re.I)
|
|
162
|
+
for item in items:
|
|
163
|
+
item = item.strip()
|
|
164
|
+
if not item:
|
|
165
|
+
continue
|
|
166
|
+
sm = _STAR_ITEM.match(item)
|
|
167
|
+
if sm:
|
|
168
|
+
star_except = {
|
|
169
|
+
n.strip().strip("`").lower()
|
|
170
|
+
for n in (sm.group(1) or "").split(",")
|
|
171
|
+
if n.strip()
|
|
172
|
+
}
|
|
173
|
+
continue
|
|
174
|
+
am = _TRAILING_ALIAS.search(item)
|
|
175
|
+
if am:
|
|
176
|
+
names.add(am.group(1).lower())
|
|
177
|
+
continue
|
|
178
|
+
im = _SIMPLE_IDENT.match(item)
|
|
179
|
+
if im:
|
|
180
|
+
names.add((im.group(1) or item.strip("`")).lower())
|
|
181
|
+
if star_except is not None and resolver is not None and _depth < 3:
|
|
182
|
+
rm = _SINGLE_REF_FROM.match(from_clause.strip())
|
|
183
|
+
if rm:
|
|
184
|
+
ref_name = rm.group(1)
|
|
185
|
+
_seen = _seen or set()
|
|
186
|
+
if ref_name not in _seen:
|
|
187
|
+
_seen.add(ref_name)
|
|
188
|
+
upstream = resolver(ref_name)
|
|
189
|
+
if upstream is not None:
|
|
190
|
+
up_names = _known_output_columns(
|
|
191
|
+
_strip_comments(upstream), resolver, _depth + 1, _seen
|
|
192
|
+
)
|
|
193
|
+
names |= up_names - star_except
|
|
194
|
+
return names
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _documented_keys(config):
|
|
198
|
+
"""Top-level keys of the columns:{} block, lowercased; empty set if none."""
|
|
199
|
+
cm = re.search(r"\bcolumns\s*:\s*({)", config)
|
|
200
|
+
block = _balanced_block(config, cm.start(1)) if cm else None
|
|
201
|
+
if not block:
|
|
202
|
+
return set()
|
|
203
|
+
keys = set()
|
|
204
|
+
for item in _split_top_level(block[0]):
|
|
205
|
+
km = re.match(r'\s*"?([A-Za-z_]\w*)"?\s*:', item)
|
|
206
|
+
if km:
|
|
207
|
+
keys.add(km.group(1).lower())
|
|
208
|
+
return keys
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def lint_text(text, path, config: Config | None = None, resolver=None):
|
|
212
|
+
cfg = config or Config()
|
|
213
|
+
findings: list[Finding] = []
|
|
214
|
+
path = path.replace("\\", "/")
|
|
215
|
+
stem = re.sub(r"\.sqlx$", "", path.rsplit("/", 1)[-1])
|
|
216
|
+
file_disabled = set(re.findall(r"sqlx-lint:\s*disable-file=([EW]\d+)", text))
|
|
217
|
+
lines = text.split("\n")
|
|
218
|
+
|
|
219
|
+
def suppressed(code, line):
|
|
220
|
+
if code in file_disabled:
|
|
221
|
+
return True
|
|
222
|
+
return (
|
|
223
|
+
line - 1 < len(lines)
|
|
224
|
+
and f"disable={code}" in lines[line - 1]
|
|
225
|
+
and "sqlx-lint:" in lines[line - 1]
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
def add(code, severity, line, message):
|
|
229
|
+
if cfg.rule_on(code) and not suppressed(code, line):
|
|
230
|
+
findings.append(Finding(code, severity, line, message))
|
|
231
|
+
|
|
232
|
+
# --- E001: locate and parse config block ---
|
|
233
|
+
m = re.search(r"\bconfig\s*({)", text)
|
|
234
|
+
if not m:
|
|
235
|
+
add("E001", "error", 1, "no config {} block found")
|
|
236
|
+
return findings
|
|
237
|
+
block = _balanced_block(text, m.start(1))
|
|
238
|
+
if block is None:
|
|
239
|
+
add(
|
|
240
|
+
"E001",
|
|
241
|
+
"error",
|
|
242
|
+
_line_of(text, m.start()),
|
|
243
|
+
"config {} block braces are unbalanced",
|
|
244
|
+
)
|
|
245
|
+
return findings
|
|
246
|
+
config_body, config_end = block
|
|
247
|
+
config_line = _line_of(text, m.start())
|
|
248
|
+
body = text[config_end:]
|
|
249
|
+
body_offset = config_end
|
|
250
|
+
|
|
251
|
+
ctype = _config_value(config_body, "type") or ""
|
|
252
|
+
schema = _config_value(config_body, "schema")
|
|
253
|
+
name = _config_value(config_body, "name")
|
|
254
|
+
documented = _documented_keys(config_body)
|
|
255
|
+
|
|
256
|
+
# --- E002: columns documentation ---
|
|
257
|
+
if ctype in cfg.documented_types and not documented:
|
|
258
|
+
add(
|
|
259
|
+
"E002",
|
|
260
|
+
"error",
|
|
261
|
+
config_line,
|
|
262
|
+
f'type "{ctype}" requires a non-empty columns: {{}} block',
|
|
263
|
+
)
|
|
264
|
+
|
|
265
|
+
# --- E003: schema suffix (declarations exempt: raw datasets may carry one) ---
|
|
266
|
+
suffix_re = "|".join(re.escape(s) for s in cfg.schema_suffixes)
|
|
267
|
+
if (
|
|
268
|
+
schema
|
|
269
|
+
and cfg.schema_suffixes
|
|
270
|
+
and ctype != "declaration"
|
|
271
|
+
and re.search(rf"(?:{suffix_re})$", schema)
|
|
272
|
+
):
|
|
273
|
+
sm = re.search(r'\bschema\s*:\s*"', config_body)
|
|
274
|
+
base = re.sub(rf"(?:{suffix_re})$", "", schema)
|
|
275
|
+
add(
|
|
276
|
+
"E003",
|
|
277
|
+
"error",
|
|
278
|
+
config_line + config_body[: sm.start()].count("\n"),
|
|
279
|
+
f'schema: "{schema}" hardcodes an environment suffix; use the '
|
|
280
|
+
f'base name ("{base}") and let --schema-suffix append it',
|
|
281
|
+
)
|
|
282
|
+
|
|
283
|
+
# --- E004: redundant name (declarations conventionally repeat it) ---
|
|
284
|
+
if name and ctype != "declaration" and name == stem:
|
|
285
|
+
nm = re.search(r'\bname\s*:\s*"', config_body)
|
|
286
|
+
add(
|
|
287
|
+
"E004",
|
|
288
|
+
"error",
|
|
289
|
+
config_line + config_body[: nm.start()].count("\n"),
|
|
290
|
+
f'name: "{name}" matches the filename and is redundant — remove it',
|
|
291
|
+
)
|
|
292
|
+
|
|
293
|
+
# --- E005 (opt-in): schema on operations/assertions ---
|
|
294
|
+
# hasOutput: true operations are exempt: schema+name define ${self()}.
|
|
295
|
+
has_output = re.search(r"\bhasOutput\s*:\s*true\b", config_body) is not None
|
|
296
|
+
if schema and ctype in NO_SCHEMA_TYPES and not has_output:
|
|
297
|
+
add(
|
|
298
|
+
"E005",
|
|
299
|
+
"error",
|
|
300
|
+
config_line,
|
|
301
|
+
f'type "{ctype}" must not set schema: — '
|
|
302
|
+
"it uses the workflow_settings.yaml default",
|
|
303
|
+
)
|
|
304
|
+
|
|
305
|
+
# --- E006: hardcoded table paths in the SQL body ---
|
|
306
|
+
clean_body = _strip_comments(body)
|
|
307
|
+
for pm in re.finditer(r"`([A-Za-z][\w-]*\.[A-Za-z]\w*\.[A-Za-z]\w*)`", clean_body):
|
|
308
|
+
if "${" in pm.group(0):
|
|
309
|
+
continue
|
|
310
|
+
add(
|
|
311
|
+
"E006",
|
|
312
|
+
"error",
|
|
313
|
+
_line_of(text, body_offset + pm.start()),
|
|
314
|
+
f"hardcoded table path `{pm.group(1)}` — "
|
|
315
|
+
"declare a source and use ${ref()}",
|
|
316
|
+
)
|
|
317
|
+
|
|
318
|
+
# --- E007: directory policies ---
|
|
319
|
+
for policy in cfg.dir_policies:
|
|
320
|
+
if policy.path_contains not in path:
|
|
321
|
+
continue
|
|
322
|
+
if policy.require_prefix and not stem.startswith(policy.require_prefix):
|
|
323
|
+
add(
|
|
324
|
+
"E007",
|
|
325
|
+
policy.severity,
|
|
326
|
+
1,
|
|
327
|
+
f"files in {policy.path_contains} must be prefixed "
|
|
328
|
+
f'"{policy.require_prefix}" (got "{stem}")',
|
|
329
|
+
)
|
|
330
|
+
if policy.require_types and ctype not in policy.require_types:
|
|
331
|
+
add(
|
|
332
|
+
"E007",
|
|
333
|
+
policy.severity,
|
|
334
|
+
config_line,
|
|
335
|
+
f"files in {policy.path_contains} must be type "
|
|
336
|
+
f'{" or ".join(policy.require_types)} (got "{ctype}")',
|
|
337
|
+
)
|
|
338
|
+
|
|
339
|
+
# --- E010: columns coverage (skip when E002 already owns the file) ---
|
|
340
|
+
in_scope = not cfg.coverage_paths or any(
|
|
341
|
+
p in path for p in cfg.coverage_paths
|
|
342
|
+
)
|
|
343
|
+
if ctype in ("table", "incremental", "view") and documented and in_scope:
|
|
344
|
+
known = _known_output_columns(clean_body, resolver)
|
|
345
|
+
missing = sorted(known - documented)
|
|
346
|
+
if missing:
|
|
347
|
+
parsed = _main_projection(clean_body)
|
|
348
|
+
sel_line = (
|
|
349
|
+
_line_of(text, body_offset + parsed[2]) if parsed else config_line
|
|
350
|
+
)
|
|
351
|
+
shown = ", ".join(f'"{n}"' for n in missing[:10])
|
|
352
|
+
more = f" (+{len(missing) - 10} more)" if len(missing) > 10 else ""
|
|
353
|
+
add(
|
|
354
|
+
"E010",
|
|
355
|
+
"error",
|
|
356
|
+
sel_line,
|
|
357
|
+
f"columns: {{}} is missing documentation for output "
|
|
358
|
+
f"column(s): {shown}{more}",
|
|
359
|
+
)
|
|
360
|
+
|
|
361
|
+
# --- W008 (opt-in): post_operations placement ---
|
|
362
|
+
pm = re.search(r"\bpost_operations\s*{", body)
|
|
363
|
+
if pm:
|
|
364
|
+
sm = re.search(r"(?im)^\s*select\b", _strip_comments(body[: pm.start()]))
|
|
365
|
+
if sm is None:
|
|
366
|
+
add(
|
|
367
|
+
"W008",
|
|
368
|
+
"warning",
|
|
369
|
+
_line_of(text, body_offset + pm.start()),
|
|
370
|
+
"post_operations {} placed before the main SELECT statement",
|
|
371
|
+
)
|
|
372
|
+
|
|
373
|
+
return findings
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def lint_file(path, config: Config | None = None, resolver=None):
|
|
377
|
+
with open(path, encoding="utf-8") as fh:
|
|
378
|
+
return lint_text(fh.read(), path, config=config, resolver=resolver)
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dataform-sqlx-lint
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Convention linter for Dataform .sqlx files — checks the config-block and project conventions that SQL linters cannot see
|
|
5
|
+
Author-email: Ivan Histand <ihistand@rotoplas.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/acuantia/dataform-sqlx-lint
|
|
8
|
+
Keywords: dataform,sqlx,lint,bigquery,pre-commit
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Environment :: Console
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
13
|
+
Requires-Python: >=3.11
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
License-File: LICENSE
|
|
16
|
+
Dynamic: license-file
|
|
17
|
+
|
|
18
|
+
# dataform-sqlx-lint
|
|
19
|
+
|
|
20
|
+
A convention linter for [Dataform](https://cloud.google.com/dataform) `.sqlx`
|
|
21
|
+
files. SQL linters (sqlfluff) check the SQL body; `dataform compile` checks
|
|
22
|
+
syntax. Neither sees the **config-block and project conventions** that keep a
|
|
23
|
+
Dataform repo healthy — this tool does.
|
|
24
|
+
|
|
25
|
+
Zero dependencies (Python ≥ 3.11 standard library only). Designed for
|
|
26
|
+
[pre-commit](https://pre-commit.com).
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
pip install dataform-sqlx-lint
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
## Rules
|
|
33
|
+
|
|
34
|
+
| Code | Default | Checks |
|
|
35
|
+
|------|---------|--------|
|
|
36
|
+
| E001 | on | `config {}` block present and balanced |
|
|
37
|
+
| E002 | on | non-empty `columns: {}` documentation on tables/views/incrementals/declarations |
|
|
38
|
+
| E003 | on | `schema:` must not hardcode an environment suffix (`_prod`/`_dev` by default) — `--schema-suffix` appends it, so a literal doubles up (`looker_prod_prod`) |
|
|
39
|
+
| E004 | on | `name:` matching the filename is redundant (declarations exempt) |
|
|
40
|
+
| E005 | opt-in | operations/assertions must not set `schema:` (`hasOutput: true` operations exempt — schema+name define `${self()}`) |
|
|
41
|
+
| E006 | on | hardcoded `` `project.dataset.table` `` paths instead of `${ref()}` — these silently break Dataform's dependency graph |
|
|
42
|
+
| E007 | on* | configurable per-directory naming/type policies (*no-op until policies are configured) |
|
|
43
|
+
| W008 | opt-in | `post_operations {}` placed before the main SELECT (style preference; Dataform accepts either) |
|
|
44
|
+
| E010 | on | every determinable output column appears in `columns: {}` — parses the main SELECT conservatively (unparseable expressions are skipped, never false-flagged) and follows `select *` through a single plain `${ref()}` into the upstream file |
|
|
45
|
+
|
|
46
|
+
Why E010 matters: `columns: {}` is what Dataform writes to BigQuery column
|
|
47
|
+
descriptions — the metadata data catalogs, BI tools, and AI/conversational
|
|
48
|
+
analytics agents read. Partial blocks leave silent gaps.
|
|
49
|
+
|
|
50
|
+
## Usage
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
dataform-sqlx-lint definitions/output/my_table.sqlx [...]
|
|
54
|
+
# exit 0 = clean or warnings only; 1 = errors
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
Run from the repo root so `--definitions-root` (default `./definitions`) can
|
|
58
|
+
index `${ref()}` targets for E010's star-resolution.
|
|
59
|
+
|
|
60
|
+
### pre-commit
|
|
61
|
+
|
|
62
|
+
```yaml
|
|
63
|
+
repos:
|
|
64
|
+
- repo: https://github.com/acuantia/dataform-sqlx-lint
|
|
65
|
+
rev: v0.1.0
|
|
66
|
+
hooks:
|
|
67
|
+
- id: dataform-sqlx-lint
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
### Configuration
|
|
71
|
+
|
|
72
|
+
`.sqlx-lint.toml` in the repo root, or a `[tool.sqlx-lint]` table in
|
|
73
|
+
`pyproject.toml` (the standalone file wins). All keys optional:
|
|
74
|
+
|
|
75
|
+
```toml
|
|
76
|
+
schema_suffixes = ["_prod", "_dev"] # E003 suffix list ([] disables)
|
|
77
|
+
documented_types = ["table", "view", "incremental", "declaration"] # E002
|
|
78
|
+
coverage_paths = ["definitions/output/"] # E010 scope; empty = everywhere
|
|
79
|
+
enable = ["E005", "W008"] # switch on opt-in rules
|
|
80
|
+
disable = ["E004"] # switch off default rules
|
|
81
|
+
|
|
82
|
+
[[dir_policies]] # E007 (repeatable)
|
|
83
|
+
path_contains = "definitions/output/looker/"
|
|
84
|
+
require_prefix = "looker_"
|
|
85
|
+
require_types = ["table", "incremental"]
|
|
86
|
+
severity = "error" # or "warning"
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
See `examples/acuantia.sqlx-lint.toml` for a complete real-world config.
|
|
90
|
+
|
|
91
|
+
### Suppressing findings
|
|
92
|
+
|
|
93
|
+
```sql
|
|
94
|
+
from `proj.raw_api.events` -- sqlx-lint: disable=E006 (declaration repoints at cutover)
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
or file-wide, anywhere in the file:
|
|
98
|
+
|
|
99
|
+
```sql
|
|
100
|
+
-- sqlx-lint: disable-file=E006
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
Suppress with a reason, sparingly — the convention is usually the fix.
|
|
104
|
+
|
|
105
|
+
## Design notes
|
|
106
|
+
|
|
107
|
+
- **Conservative by construction**: the SQL projection parser only claims
|
|
108
|
+
column names it can determine (aliases, simple identifiers, resolvable
|
|
109
|
+
`select *`); anything ambiguous is skipped, so E010 never false-flags.
|
|
110
|
+
- **Declarations are exempt** from E003/E004 deliberately: raw source datasets
|
|
111
|
+
legitimately carry environment-suffixed names, and Dataform requires `name:`
|
|
112
|
+
on declarations.
|
|
113
|
+
- Rule codes are stable; gaps in the numbering are historical.
|
|
114
|
+
|
|
115
|
+
## Development
|
|
116
|
+
|
|
117
|
+
```bash
|
|
118
|
+
python3 -m venv .venv && .venv/bin/pip install -e . pytest
|
|
119
|
+
.venv/bin/pytest # 53 tests
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
## License
|
|
123
|
+
|
|
124
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
dataform_sqlx_lint/__init__.py,sha256=rrjJ-9Htqn9zU92rxLEJWBGe4Fo6YMLcYu086V8ithY,281
|
|
2
|
+
dataform_sqlx_lint/cli.py,sha256=lrrzJHlT8bfwy-bfkMEFGrOwJpF7H2R21aJK1QZTc0A,2426
|
|
3
|
+
dataform_sqlx_lint/config.py,sha256=gorcj1Rv1nPiQVyMXlLOItGsjCSxVUD6ymVvPz5s3tI,3624
|
|
4
|
+
dataform_sqlx_lint/linter.py,sha256=RFZUShjN5nSa0C-m-4jPOBfMKlVehFTIxUeg4_RDQ0Q,13089
|
|
5
|
+
dataform_sqlx_lint-0.1.0.dist-info/licenses/LICENSE,sha256=LwjbOvYxlORpP0d7u3F_KRTjhPzP04XblKIKuTtC3gw,1065
|
|
6
|
+
dataform_sqlx_lint-0.1.0.dist-info/METADATA,sha256=jPy56Yhrx0T_BBN3rE1KJZsUbozP7e7qiy_Wcy5gTqE,4744
|
|
7
|
+
dataform_sqlx_lint-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
8
|
+
dataform_sqlx_lint-0.1.0.dist-info/entry_points.txt,sha256=uoj3vFX4zQpXe6QO6alBfbWjSpF8zNiot6qyESmlAFI,73
|
|
9
|
+
dataform_sqlx_lint-0.1.0.dist-info/top_level.txt,sha256=xsaISCF9odSTXlB9lZwyuHW3cGF3BpznttdE88XkaPA,19
|
|
10
|
+
dataform_sqlx_lint-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Acuantia
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
dataform_sqlx_lint
|