sqlinclude 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sqlinclude/__init__.py +7 -0
- sqlinclude/__main__.py +8 -0
- sqlinclude/cli.py +226 -0
- sqlinclude/editor.py +179 -0
- sqlinclude/preprocess.py +185 -0
- sqlinclude-0.1.0.dist-info/METADATA +145 -0
- sqlinclude-0.1.0.dist-info/RECORD +11 -0
- sqlinclude-0.1.0.dist-info/WHEEL +5 -0
- sqlinclude-0.1.0.dist-info/entry_points.txt +2 -0
- sqlinclude-0.1.0.dist-info/licenses/LICENSE +21 -0
- sqlinclude-0.1.0.dist-info/top_level.txt +1 -0
sqlinclude/__init__.py
ADDED
sqlinclude/__main__.py
ADDED
sqlinclude/cli.py
ADDED
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
"""Command-line interface for sqlinclude."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import os
|
|
7
|
+
import re
|
|
8
|
+
import sys
|
|
9
|
+
|
|
10
|
+
from . import __version__
|
|
11
|
+
from .editor import (
|
|
12
|
+
edit_buffer,
|
|
13
|
+
have_ctrl_tty,
|
|
14
|
+
parse_buffer,
|
|
15
|
+
render_buffer,
|
|
16
|
+
resolve_editor,
|
|
17
|
+
undefined_count,
|
|
18
|
+
)
|
|
19
|
+
from .preprocess import (
|
|
20
|
+
ORIGIN_UNDEFINED,
|
|
21
|
+
STDIN_NAME,
|
|
22
|
+
collect_tree,
|
|
23
|
+
collect_vars,
|
|
24
|
+
render_tree,
|
|
25
|
+
walk,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _stdout_can_encode(text: str) -> bool:
|
|
30
|
+
enc = getattr(sys.stdout, "encoding", None) or "ascii"
|
|
31
|
+
try:
|
|
32
|
+
text.encode(enc)
|
|
33
|
+
except (UnicodeEncodeError, LookupError):
|
|
34
|
+
return False
|
|
35
|
+
return True
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
39
|
+
parser = argparse.ArgumentParser(
|
|
40
|
+
prog="sqlinclude",
|
|
41
|
+
description="Expand @include directives and @define variables in SQL source "
|
|
42
|
+
"(no SQL parsing).",
|
|
43
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
44
|
+
epilog=(
|
|
45
|
+
"example (analysis.sql):\n"
|
|
46
|
+
"\n"
|
|
47
|
+
" @define start = '2024-01-01'\n"
|
|
48
|
+
" @define end = '2024-12-31'\n"
|
|
49
|
+
"\n"
|
|
50
|
+
" WITH users AS (\n"
|
|
51
|
+
' @include "users.sql"\n'
|
|
52
|
+
" ),\n"
|
|
53
|
+
"\n"
|
|
54
|
+
" orders AS (\n"
|
|
55
|
+
" @include orders.sql\n"
|
|
56
|
+
" )\n"
|
|
57
|
+
"\n"
|
|
58
|
+
" SELECT ... WHERE created BETWEEN @start AND @end\n"
|
|
59
|
+
"\n"
|
|
60
|
+
"run:\n"
|
|
61
|
+
"\n"
|
|
62
|
+
" sqlinclude analysis.sql | bq query\n"
|
|
63
|
+
" sqlinclude --vars start='2024-06-01' analysis.sql | bq query\n"
|
|
64
|
+
" sqlinclude --tree analysis.sql # show the include tree, not SQL\n"
|
|
65
|
+
"\n"
|
|
66
|
+
"variables:\n"
|
|
67
|
+
"\n"
|
|
68
|
+
" @define name = value set a variable in a file (first definition wins)\n"
|
|
69
|
+
" --vars name=value set a variable from the command line (repeatable)\n"
|
|
70
|
+
" @name substituted where defined; undefined names pass\n"
|
|
71
|
+
" through unchanged (BigQuery @params stay intact)\n"
|
|
72
|
+
" -e, --edit open the editor even when nothing is undefined\n"
|
|
73
|
+
" --no-edit never open the editor; no stderr notes\n"
|
|
74
|
+
"\n"
|
|
75
|
+
"editing:\n"
|
|
76
|
+
"\n"
|
|
77
|
+
" When a variable is undefined, sqlinclude opens $VISUAL/$EDITOR (or vim,\n"
|
|
78
|
+
" notepad on Windows) on the controlling terminal, even if its output is\n"
|
|
79
|
+
" piped. Edit values; delete a line to leave that variable undefined; blank\n"
|
|
80
|
+
" and # lines are ignored. With -e/--edit the buffer also shows the @include\n"
|
|
81
|
+
" tree. Undefined variables are always reported on stderr -- use\n"
|
|
82
|
+
" --no-edit to silence everything.\n"
|
|
83
|
+
),
|
|
84
|
+
)
|
|
85
|
+
parser.add_argument(
|
|
86
|
+
"file",
|
|
87
|
+
nargs="?",
|
|
88
|
+
default="-",
|
|
89
|
+
help="SQL file to preprocess (default: read stdin)",
|
|
90
|
+
)
|
|
91
|
+
parser.add_argument(
|
|
92
|
+
"-n",
|
|
93
|
+
"--no-markers",
|
|
94
|
+
action="store_true",
|
|
95
|
+
help="do not emit -- #line markers around includes",
|
|
96
|
+
)
|
|
97
|
+
parser.add_argument(
|
|
98
|
+
"--vars",
|
|
99
|
+
action="append",
|
|
100
|
+
default=[],
|
|
101
|
+
metavar="name=value",
|
|
102
|
+
help="set a variable (repeatable); wins over @define",
|
|
103
|
+
)
|
|
104
|
+
parser.add_argument(
|
|
105
|
+
"-e",
|
|
106
|
+
"--edit",
|
|
107
|
+
action="store_true",
|
|
108
|
+
help="open the editor even when no variable is undefined",
|
|
109
|
+
)
|
|
110
|
+
parser.add_argument(
|
|
111
|
+
"--no-edit",
|
|
112
|
+
action="store_true",
|
|
113
|
+
help="never open the editor; leave undefined @names as-is (no stderr notes)",
|
|
114
|
+
)
|
|
115
|
+
parser.add_argument(
|
|
116
|
+
"-t",
|
|
117
|
+
"--tree",
|
|
118
|
+
action="store_true",
|
|
119
|
+
help="print the @include dependency tree as ASCII on stdout instead of expanding SQL",
|
|
120
|
+
)
|
|
121
|
+
parser.add_argument(
|
|
122
|
+
"--ascii",
|
|
123
|
+
action="store_true",
|
|
124
|
+
help="use ASCII (not Unicode) box characters in the include tree",
|
|
125
|
+
)
|
|
126
|
+
parser.add_argument(
|
|
127
|
+
"--version",
|
|
128
|
+
action="version",
|
|
129
|
+
version=f"%(prog)s {__version__}",
|
|
130
|
+
)
|
|
131
|
+
return parser
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def main(argv: list[str] | None = None) -> int:
|
|
135
|
+
if argv is None:
|
|
136
|
+
argv = sys.argv
|
|
137
|
+
args = build_parser().parse_args(argv[1:])
|
|
138
|
+
|
|
139
|
+
cli_vars: dict[str, str] = {}
|
|
140
|
+
for pair in args.vars:
|
|
141
|
+
if "=" not in pair:
|
|
142
|
+
sys.stderr.write(f"sqlinclude: --vars expects name=value, got {pair!r}\n")
|
|
143
|
+
return 2
|
|
144
|
+
name, value = pair.split("=", 1)
|
|
145
|
+
if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", name):
|
|
146
|
+
sys.stderr.write(f"sqlinclude: invalid variable name in --vars: {name!r}\n")
|
|
147
|
+
return 2
|
|
148
|
+
cli_vars[name] = value
|
|
149
|
+
|
|
150
|
+
if args.file == "-":
|
|
151
|
+
source = sys.stdin.read()
|
|
152
|
+
top = None
|
|
153
|
+
else:
|
|
154
|
+
try:
|
|
155
|
+
with open(args.file, encoding="utf-8") as fh:
|
|
156
|
+
source = fh.read()
|
|
157
|
+
except OSError as exc:
|
|
158
|
+
sys.stderr.write(f"sqlinclude: {args.file}: {exc}\n")
|
|
159
|
+
return 1
|
|
160
|
+
top = args.file
|
|
161
|
+
if source and not source.endswith("\n"):
|
|
162
|
+
source += "\n"
|
|
163
|
+
|
|
164
|
+
top_disp = (
|
|
165
|
+
os.path.relpath(os.path.realpath(top), os.getcwd()).replace("\\", "/")
|
|
166
|
+
if top
|
|
167
|
+
else STDIN_NAME
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
if args.tree:
|
|
171
|
+
root = collect_tree(source, top)
|
|
172
|
+
if root is None:
|
|
173
|
+
return 1
|
|
174
|
+
ascii_only = args.ascii or not _stdout_can_encode("└──")
|
|
175
|
+
sys.stdout.write(render_tree(root, ascii_only=ascii_only))
|
|
176
|
+
return 0
|
|
177
|
+
|
|
178
|
+
collected = collect_vars(source, top, cli_vars)
|
|
179
|
+
if collected is None:
|
|
180
|
+
return 1
|
|
181
|
+
order, values, sources = collected
|
|
182
|
+
undefined_present = any(values[n] is None for n in order)
|
|
183
|
+
|
|
184
|
+
if (
|
|
185
|
+
top is not None
|
|
186
|
+
and not args.no_edit
|
|
187
|
+
and (undefined_present or args.edit)
|
|
188
|
+
and have_ctrl_tty()
|
|
189
|
+
):
|
|
190
|
+
editor = resolve_editor()
|
|
191
|
+
if undefined_present:
|
|
192
|
+
sys.stderr.write(
|
|
193
|
+
f"sqlinclude: opening {editor} for {undefined_count(order, values)} "
|
|
194
|
+
"undefined variable(s)\n"
|
|
195
|
+
)
|
|
196
|
+
tree = None
|
|
197
|
+
if args.edit:
|
|
198
|
+
tree = collect_tree(source, top)
|
|
199
|
+
if tree is None:
|
|
200
|
+
return 1
|
|
201
|
+
edited = edit_buffer(render_buffer(top_disp, order, values, sources, tree), editor)
|
|
202
|
+
if edited is None:
|
|
203
|
+
return 1
|
|
204
|
+
env = parse_buffer(edited)
|
|
205
|
+
blocked = set(order) - set(env)
|
|
206
|
+
else:
|
|
207
|
+
env = dict(cli_vars)
|
|
208
|
+
blocked = set()
|
|
209
|
+
if not args.no_edit and undefined_present:
|
|
210
|
+
names = [n for n in order if values[n] is None]
|
|
211
|
+
sys.stderr.write(
|
|
212
|
+
"sqlinclude: undefined variables "
|
|
213
|
+
f"({len(names)}): {', '.join('@' + n for n in names)} "
|
|
214
|
+
"-- left as @name\n"
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
out: list[str] = []
|
|
218
|
+
stack: list[tuple[str, str]] = []
|
|
219
|
+
if walk(source, top, out, stack, args.no_markers, env, blocked):
|
|
220
|
+
return 1
|
|
221
|
+
sys.stdout.write("".join(out))
|
|
222
|
+
return 0
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
if __name__ == "__main__":
|
|
226
|
+
sys.exit(main())
|
sqlinclude/editor.py
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
"""Cross-platform terminal + editor helpers.
|
|
2
|
+
|
|
3
|
+
The interactive undefined-variable flow needs to reach the user's controlling
|
|
4
|
+
terminal even when stdout is piped (e.g. ``sqlinclude x.sql | bq query``). On
|
|
5
|
+
POSIX that is ``/dev/tty``; on Windows the console is reached through the
|
|
6
|
+
``CONIN$`` / ``CONOUT$`` device names. This module hides that difference.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import contextlib
|
|
12
|
+
import os
|
|
13
|
+
import shlex
|
|
14
|
+
import subprocess
|
|
15
|
+
import sys
|
|
16
|
+
import tempfile
|
|
17
|
+
from typing import Iterator
|
|
18
|
+
|
|
19
|
+
from .preprocess import (
|
|
20
|
+
ORIGIN_UNDEFINED,
|
|
21
|
+
DEFINE_RE,
|
|
22
|
+
render_tree,
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
IS_WINDOWS = os.name == "nt"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def render_buffer(
|
|
29
|
+
top_disp: str,
|
|
30
|
+
order: list[str],
|
|
31
|
+
values: dict[str, str | None],
|
|
32
|
+
origins: dict[str, str],
|
|
33
|
+
tree: dict | None = None,
|
|
34
|
+
) -> str:
|
|
35
|
+
lines = [
|
|
36
|
+
f"# sqlinclude variables -- {top_disp}",
|
|
37
|
+
"# Edit values below. Delete a line to leave that variable undefined",
|
|
38
|
+
"# (it passes through as @name). Blank lines and # comments are ignored.",
|
|
39
|
+
"#",
|
|
40
|
+
]
|
|
41
|
+
if tree is not None:
|
|
42
|
+
lines.append("# include tree:")
|
|
43
|
+
lines.extend(f"# {ln}" for ln in render_tree(tree).splitlines())
|
|
44
|
+
lines.append("#")
|
|
45
|
+
for name in order:
|
|
46
|
+
origin = origins.get(name, ORIGIN_UNDEFINED)
|
|
47
|
+
lines.append(f"# {origin}")
|
|
48
|
+
lines.append(f"@define {name} = {values.get(name) or ''}")
|
|
49
|
+
return "\n".join(lines) + "\n"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def parse_buffer(text: str) -> dict[str, str]:
|
|
53
|
+
"""Read an edited buffer; empty values mean 'undefined'. Returns env dict."""
|
|
54
|
+
env: dict[str, str] = {}
|
|
55
|
+
for line in text.splitlines():
|
|
56
|
+
s = line.strip()
|
|
57
|
+
if not s or s.startswith("#"):
|
|
58
|
+
continue
|
|
59
|
+
m = DEFINE_RE.match(s)
|
|
60
|
+
if m:
|
|
61
|
+
name, value = m.group(1), m.group(2)
|
|
62
|
+
if value:
|
|
63
|
+
env[name] = value
|
|
64
|
+
return env
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def resolve_editor() -> str:
|
|
68
|
+
"""Honor $VISUAL / $EDITOR; fall back to a platform-appropriate editor."""
|
|
69
|
+
editor = os.environ.get("VISUAL") or os.environ.get("EDITOR")
|
|
70
|
+
if editor:
|
|
71
|
+
return editor
|
|
72
|
+
return "notepad" if IS_WINDOWS else "vim"
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def split_editor(editor: str) -> list[str]:
|
|
76
|
+
"""Split an editor command into argv.
|
|
77
|
+
|
|
78
|
+
POSIX shell rules apply off Windows; on Windows backslashes in paths must
|
|
79
|
+
survive, so we use ``shlex.split(..., posix=False)`` and strip the quotes
|
|
80
|
+
it leaves behind.
|
|
81
|
+
"""
|
|
82
|
+
if not IS_WINDOWS:
|
|
83
|
+
return shlex.split(editor)
|
|
84
|
+
parts = shlex.split(editor, posix=False)
|
|
85
|
+
return [
|
|
86
|
+
p[1:-1] if len(p) >= 2 and p[0] == p[-1] and p[0] in "\"'" else p for p in parts
|
|
87
|
+
]
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def have_ctrl_tty() -> bool:
|
|
91
|
+
"""True if a controlling terminal can be opened."""
|
|
92
|
+
if IS_WINDOWS:
|
|
93
|
+
try:
|
|
94
|
+
with open("CONIN$", "rb"):
|
|
95
|
+
return True
|
|
96
|
+
except OSError:
|
|
97
|
+
return False
|
|
98
|
+
try:
|
|
99
|
+
fd = os.open("/dev/tty", os.O_RDWR)
|
|
100
|
+
except OSError:
|
|
101
|
+
return False
|
|
102
|
+
os.close(fd)
|
|
103
|
+
return True
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
@contextlib.contextmanager
|
|
107
|
+
def _console_streams() -> Iterator[tuple | None]:
|
|
108
|
+
"""Yield ``(stdin, stdout, stderr)`` attached to the controlling console.
|
|
109
|
+
|
|
110
|
+
Yields ``None`` when no console is available. On Windows the input and
|
|
111
|
+
output devices are distinct objects; on POSIX the same tty fd serves all
|
|
112
|
+
three streams.
|
|
113
|
+
"""
|
|
114
|
+
if IS_WINDOWS:
|
|
115
|
+
try:
|
|
116
|
+
stdin = open("CONIN$", "rb")
|
|
117
|
+
stdout = open("CONOUT$", "wb")
|
|
118
|
+
stderr = open("CONOUT$", "wb")
|
|
119
|
+
except OSError:
|
|
120
|
+
yield None
|
|
121
|
+
return
|
|
122
|
+
try:
|
|
123
|
+
yield (stdin, stdout, stderr)
|
|
124
|
+
finally:
|
|
125
|
+
for handle in (stdin, stdout, stderr):
|
|
126
|
+
with contextlib.suppress(OSError):
|
|
127
|
+
handle.close()
|
|
128
|
+
else:
|
|
129
|
+
try:
|
|
130
|
+
fd = os.open("/dev/tty", os.O_RDWR)
|
|
131
|
+
except OSError:
|
|
132
|
+
yield None
|
|
133
|
+
return
|
|
134
|
+
try:
|
|
135
|
+
yield (fd, fd, fd)
|
|
136
|
+
finally:
|
|
137
|
+
with contextlib.suppress(OSError):
|
|
138
|
+
os.close(fd)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def undefined_count(order: list[str], values: dict[str, str | None]) -> int:
|
|
142
|
+
return sum(1 for n in order if values[n] is None)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def edit_buffer(text: str, editor: str) -> str | None:
|
|
146
|
+
"""Open ``editor`` on a temp file holding ``text``; return edited contents.
|
|
147
|
+
|
|
148
|
+
Returns ``None`` if the editor exits non-zero.
|
|
149
|
+
"""
|
|
150
|
+
fd, tmp = tempfile.mkstemp(prefix="sqlinclude-", suffix=".sql")
|
|
151
|
+
try:
|
|
152
|
+
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
|
153
|
+
fh.write(text)
|
|
154
|
+
with _console_streams() as streams:
|
|
155
|
+
if IS_WINDOWS:
|
|
156
|
+
# Editors such as VS Code resolve to ``code.cmd``; those need
|
|
157
|
+
# the command interpreter, so pass a command line and shell=True.
|
|
158
|
+
cmdline = editor + " " + subprocess.list2cmdline([tmp])
|
|
159
|
+
if streams is not None:
|
|
160
|
+
rc = subprocess.call(
|
|
161
|
+
cmdline, shell=True, stdin=streams[0], stdout=streams[1], stderr=streams[2]
|
|
162
|
+
)
|
|
163
|
+
else:
|
|
164
|
+
rc = subprocess.call(cmdline, shell=True)
|
|
165
|
+
else:
|
|
166
|
+
cmd = split_editor(editor) + [tmp]
|
|
167
|
+
if streams is not None:
|
|
168
|
+
rc = subprocess.call(
|
|
169
|
+
cmd, stdin=streams[0], stdout=streams[1], stderr=streams[2]
|
|
170
|
+
)
|
|
171
|
+
else:
|
|
172
|
+
rc = subprocess.call(cmd)
|
|
173
|
+
if rc != 0:
|
|
174
|
+
sys.stderr.write(f"sqlinclude: editor exited with status {rc}\n")
|
|
175
|
+
return None
|
|
176
|
+
with open(tmp, encoding="utf-8") as fh:
|
|
177
|
+
return fh.read()
|
|
178
|
+
finally:
|
|
179
|
+
os.unlink(tmp)
|
sqlinclude/preprocess.py
ADDED
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
"""Core @include / @define expansion.
|
|
2
|
+
|
|
3
|
+
This module is pure with respect to *policy*: it walks a source tree, records
|
|
4
|
+
variables and include relationships, and emits expanded SQL. It never decides
|
|
5
|
+
which editor to launch or how to reach a terminal -- that lives in
|
|
6
|
+
:mod:`sqlinclude.editor`.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import os
|
|
12
|
+
import re
|
|
13
|
+
import sys
|
|
14
|
+
from typing import Callable, Iterable
|
|
15
|
+
|
|
16
|
+
INCLUDE_RE = re.compile(r'^\s*@include\s+(?:"([^"]+)"|(\S+))')
|
|
17
|
+
DEFINE_RE = re.compile(r"^\s*@define\s+([A-Za-z_][A-Za-z0-9_]*)\s*=?\s*(.*?)\s*$")
|
|
18
|
+
VAR_RE = re.compile(r"@([A-Za-z_][A-Za-z0-9_]*)")
|
|
19
|
+
STDIN_NAME = "<stdin>"
|
|
20
|
+
|
|
21
|
+
ORIGIN_UNDEFINED = "undefined"
|
|
22
|
+
ORIGIN_CLI = "--vars"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _slashed(path: str) -> str:
|
|
26
|
+
"""Normalize path separators for emitted markers/tree names.
|
|
27
|
+
|
|
28
|
+
Keeps output identical across platforms (and friendly to downstream
|
|
29
|
+
parsers that dislike backslashes in ``#line`` markers).
|
|
30
|
+
"""
|
|
31
|
+
return path.replace("\\", "/")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def walk(
|
|
35
|
+
source: str,
|
|
36
|
+
cur: str | None,
|
|
37
|
+
out: list[str] | None,
|
|
38
|
+
stack: list[tuple[str, str]],
|
|
39
|
+
no_markers: bool,
|
|
40
|
+
env: dict[str, str],
|
|
41
|
+
blocked: set[str],
|
|
42
|
+
on_var: Callable[[str, str | None, str, int], None] | None = None,
|
|
43
|
+
on_define: Callable[[str, str, int], None] | None = None,
|
|
44
|
+
tree: dict | None = None,
|
|
45
|
+
) -> bool:
|
|
46
|
+
"""Process directives in ``source``.
|
|
47
|
+
|
|
48
|
+
Appends expansion to ``out`` (or only collects via callbacks when ``out``
|
|
49
|
+
is ``None``); ``tree``, when given, records include relationships as
|
|
50
|
+
``{"name", "children"}`` nodes. Returns ``True`` on error.
|
|
51
|
+
"""
|
|
52
|
+
cwd = os.getcwd()
|
|
53
|
+
cur_disp = _slashed(os.path.relpath(os.path.realpath(cur), cwd)) if cur else STDIN_NAME
|
|
54
|
+
line_no = 0
|
|
55
|
+
for raw in source.splitlines(keepends=True):
|
|
56
|
+
line_no += 1
|
|
57
|
+
m = INCLUDE_RE.match(raw)
|
|
58
|
+
if m:
|
|
59
|
+
inc = m.group(1) or m.group(2)
|
|
60
|
+
base = os.path.dirname(cur) if cur else cwd
|
|
61
|
+
path = os.path.normpath(os.path.join(base, inc))
|
|
62
|
+
canon = os.path.realpath(path)
|
|
63
|
+
stack_canons = [c for c, _ in stack]
|
|
64
|
+
|
|
65
|
+
if not os.path.isfile(canon):
|
|
66
|
+
sys.stderr.write(f"sqlinclude: {path}: no such file\n")
|
|
67
|
+
return True
|
|
68
|
+
if canon in stack_canons:
|
|
69
|
+
chain = " -> ".join(stack_canons + [canon])
|
|
70
|
+
sys.stderr.write(f"sqlinclude: include cycle: {chain}\n")
|
|
71
|
+
return True
|
|
72
|
+
|
|
73
|
+
try:
|
|
74
|
+
with open(canon, encoding="utf-8") as fh:
|
|
75
|
+
sub = fh.read()
|
|
76
|
+
except OSError as exc:
|
|
77
|
+
sys.stderr.write(f"sqlinclude: {path}: {exc}\n")
|
|
78
|
+
return True
|
|
79
|
+
if sub and not sub.endswith("\n"):
|
|
80
|
+
sub += "\n"
|
|
81
|
+
|
|
82
|
+
disp = _slashed(os.path.relpath(canon, cwd))
|
|
83
|
+
if out is not None and not no_markers:
|
|
84
|
+
out.append(f'-- #line 1 "{disp}"\n')
|
|
85
|
+
subtree = None
|
|
86
|
+
if tree is not None:
|
|
87
|
+
subtree = {"name": disp, "children": []}
|
|
88
|
+
tree["children"].append(subtree)
|
|
89
|
+
stack.append((canon, disp))
|
|
90
|
+
if walk(
|
|
91
|
+
sub, path, out, stack, no_markers, env, blocked, on_var, on_define, subtree
|
|
92
|
+
):
|
|
93
|
+
return True
|
|
94
|
+
stack.pop()
|
|
95
|
+
if out is not None and not no_markers:
|
|
96
|
+
parent = stack[-1][1] if stack else cur_disp
|
|
97
|
+
out.append(f'-- #line {line_no + 1} "{parent}"\n')
|
|
98
|
+
continue
|
|
99
|
+
|
|
100
|
+
m = DEFINE_RE.match(raw)
|
|
101
|
+
if m:
|
|
102
|
+
name, value = m.group(1), m.group(2)
|
|
103
|
+
if name not in blocked and name not in env:
|
|
104
|
+
env[name] = value
|
|
105
|
+
if on_define is not None:
|
|
106
|
+
on_define(name, cur_disp, line_no)
|
|
107
|
+
continue
|
|
108
|
+
|
|
109
|
+
if on_var is not None:
|
|
110
|
+
for name in VAR_RE.findall(raw):
|
|
111
|
+
on_var(name, env.get(name), cur_disp, line_no)
|
|
112
|
+
if out is not None:
|
|
113
|
+
out.append(VAR_RE.sub(lambda mm: env.get(mm.group(1), mm.group(0)), raw))
|
|
114
|
+
return False
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def collect_vars(
|
|
118
|
+
source: str, top: str | None, cli_vars: dict[str, str]
|
|
119
|
+
) -> tuple[list[str], dict[str, str | None], dict[str, str]] | None:
|
|
120
|
+
"""Walk the tree without emitting.
|
|
121
|
+
|
|
122
|
+
Returns ``(order, values, sources)`` where ``order`` lists each variable's
|
|
123
|
+
first occurrence, ``values`` its value at that point (``None`` if
|
|
124
|
+
undefined), and ``sources`` its origin (``"--vars"``, ``"file:line"``, or
|
|
125
|
+
``"undefined"``). Returns ``None`` on error.
|
|
126
|
+
"""
|
|
127
|
+
env = dict(cli_vars)
|
|
128
|
+
sources = {name: ORIGIN_CLI for name in cli_vars}
|
|
129
|
+
order: list[str] = []
|
|
130
|
+
values: dict[str, str | None] = {}
|
|
131
|
+
origins: dict[str, str] = {}
|
|
132
|
+
|
|
133
|
+
def on_define(name: str, disp: str, line: int) -> None:
|
|
134
|
+
sources[name] = f"{disp}:{line}"
|
|
135
|
+
|
|
136
|
+
def on_var(name: str, value: str | None, disp: str, line: int) -> None:
|
|
137
|
+
if name not in values:
|
|
138
|
+
order.append(name)
|
|
139
|
+
values[name] = value
|
|
140
|
+
origins[name] = sources.get(name, ORIGIN_UNDEFINED)
|
|
141
|
+
|
|
142
|
+
if walk(source, top, None, [], True, env, set(), on_var, on_define):
|
|
143
|
+
return None
|
|
144
|
+
return order, values, origins
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def collect_tree(source: str, top: str | None) -> dict | None:
|
|
148
|
+
"""Walk the tree, recording @include relationships.
|
|
149
|
+
|
|
150
|
+
Returns the root node ``{"name", "children"}`` or ``None`` on error.
|
|
151
|
+
"""
|
|
152
|
+
root = {
|
|
153
|
+
"name": _slashed(os.path.relpath(os.path.realpath(top), os.getcwd()))
|
|
154
|
+
if top
|
|
155
|
+
else STDIN_NAME,
|
|
156
|
+
"children": [],
|
|
157
|
+
}
|
|
158
|
+
if walk(source, top, None, [], True, {}, set(), tree=root):
|
|
159
|
+
return None
|
|
160
|
+
return root
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def render_tree(node: dict, ascii_only: bool = False) -> str:
|
|
164
|
+
"""Render an include tree as an ASCII/Unicode diagram (one line per file)."""
|
|
165
|
+
lines = [node["name"]]
|
|
166
|
+
children: Iterable[dict] = node["children"]
|
|
167
|
+
for i, child in enumerate(children):
|
|
168
|
+
_render_tree_child(child, "", i == len(children) - 1, lines, ascii_only)
|
|
169
|
+
return "\n".join(lines) + "\n"
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _render_tree_child(
|
|
173
|
+
node: dict, prefix: str, last: bool, lines: list[str], ascii_only: bool
|
|
174
|
+
) -> None:
|
|
175
|
+
if ascii_only:
|
|
176
|
+
stem = "`-- " if last else "|-- "
|
|
177
|
+
bar = "| "
|
|
178
|
+
else:
|
|
179
|
+
stem = "└── " if last else "├── "
|
|
180
|
+
bar = "│ "
|
|
181
|
+
lines.append(prefix + stem + node["name"])
|
|
182
|
+
next_prefix = prefix + (" " if last else bar)
|
|
183
|
+
children: Iterable[dict] = node["children"]
|
|
184
|
+
for i, child in enumerate(children):
|
|
185
|
+
_render_tree_child(child, next_prefix, i == len(children) - 1, lines, ascii_only)
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: sqlinclude
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Tiny SQL source preprocessor: expand @include directives and @define variables.
|
|
5
|
+
Author-email: lukasburski <lukasbursky@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/lukasbursky/sqlinclude
|
|
8
|
+
Project-URL: Source, https://github.com/lukasbursky/sqlinclude
|
|
9
|
+
Project-URL: Issues, https://github.com/lukasbursky/sqlinclude/issues
|
|
10
|
+
Keywords: sql,preprocessor,include,bigquery,templating
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Environment :: Console
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
19
|
+
Classifier: Topic :: Database
|
|
20
|
+
Classifier: Topic :: Software Development :: Pre-processors
|
|
21
|
+
Requires-Python: >=3.12
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE
|
|
24
|
+
Provides-Extra: dev
|
|
25
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
26
|
+
Requires-Dist: build>=1.2; extra == "dev"
|
|
27
|
+
Dynamic: license-file
|
|
28
|
+
|
|
29
|
+
# sqlinclude
|
|
30
|
+
|
|
31
|
+
Tiny SQL source preprocessor. Expands `@include` directives and `@define`
|
|
32
|
+
variables recursively. No SQL parsing, no opinion about any database. Intended
|
|
33
|
+
to be piped into a query tool:
|
|
34
|
+
|
|
35
|
+
```console
|
|
36
|
+
sqlinclude analysis.sql | bq query
|
|
37
|
+
bq query "$(sqlinclude analysis.sql)"
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
Written for Unix and Windows alike: it is a single dependency-free Python
|
|
41
|
+
package and installs a `sqlinclude` console command (a `sqlinclude.exe` on
|
|
42
|
+
Windows).
|
|
43
|
+
|
|
44
|
+
## Install
|
|
45
|
+
|
|
46
|
+
```console
|
|
47
|
+
pipx install sqlinclude
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
or, into the current environment:
|
|
51
|
+
|
|
52
|
+
```console
|
|
53
|
+
pip install sqlinclude
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
Requires Python 3.12+.
|
|
57
|
+
|
|
58
|
+
## Directives
|
|
59
|
+
|
|
60
|
+
Line-oriented, leading whitespace allowed:
|
|
61
|
+
|
|
62
|
+
```
|
|
63
|
+
@include file.sql
|
|
64
|
+
@include "file.sql"
|
|
65
|
+
@define name = value (the `= ` is optional; value is the rest of the
|
|
66
|
+
line, kept verbatim including quotes)
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
Include paths are resolved relative to the including file. Includes may nest;
|
|
70
|
+
cycles are detected and reported as errors.
|
|
71
|
+
|
|
72
|
+
Variables form a single environment filled in expansion order: the first
|
|
73
|
+
`@define name ...` seen anywhere in the tree wins, and later definitions of the
|
|
74
|
+
same name are ignored (so a parent file's definition always overrides an
|
|
75
|
+
included fragment's default). Definitions from an included file leak back to
|
|
76
|
+
the including file. A `@name` with no definition passes through unchanged --
|
|
77
|
+
BigQuery's native `@param` syntax is never touched.
|
|
78
|
+
|
|
79
|
+
Each include block is bracketed with `-- #line N "file"` markers (with forward
|
|
80
|
+
slashes on every platform) so error messages from downstream tools point at the
|
|
81
|
+
originating source file and line. Use `-n`/`--no-markers` to suppress them.
|
|
82
|
+
|
|
83
|
+
## Example
|
|
84
|
+
|
|
85
|
+
`analysis.sql`:
|
|
86
|
+
|
|
87
|
+
```sql
|
|
88
|
+
@define start = '2024-01-01'
|
|
89
|
+
@define end = '2024-12-31'
|
|
90
|
+
|
|
91
|
+
WITH users AS (
|
|
92
|
+
@include "users.sql"
|
|
93
|
+
),
|
|
94
|
+
|
|
95
|
+
orders AS (
|
|
96
|
+
@include orders.sql
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
SELECT ... WHERE created BETWEEN @start AND @end
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
Run:
|
|
103
|
+
|
|
104
|
+
```console
|
|
105
|
+
sqlinclude analysis.sql | bq query
|
|
106
|
+
sqlinclude --vars start='2024-06-01' analysis.sql | bq query
|
|
107
|
+
sqlinclude --tree analysis.sql # show the include tree, not SQL
|
|
108
|
+
sqlinclude --tree --ascii analysis.sql # ASCII box characters instead of Unicode
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
## Options
|
|
112
|
+
|
|
113
|
+
| Option | Description |
|
|
114
|
+
| --- | --- |
|
|
115
|
+
| `file` | SQL file to preprocess (default: read stdin) |
|
|
116
|
+
| `-n`, `--no-markers` | do not emit `-- #line` markers around includes |
|
|
117
|
+
| `--vars name=value` | set a variable (repeatable); wins over `@define` |
|
|
118
|
+
| `-e`, `--edit` | open the editor even when no variable is undefined |
|
|
119
|
+
| `--no-edit` | never open the editor; leave undefined `@name`s as-is |
|
|
120
|
+
| `-t`, `--tree` | print the `@include` dependency tree instead of SQL |
|
|
121
|
+
| `--ascii` | use ASCII box characters in the include tree |
|
|
122
|
+
| `--version` | show the version |
|
|
123
|
+
|
|
124
|
+
## Editing undefined variables
|
|
125
|
+
|
|
126
|
+
When a variable is undefined, `sqlinclude` opens the editor named by
|
|
127
|
+
`$VISUAL`/`$EDITOR` on the controlling terminal, even when its output is piped.
|
|
128
|
+
Edit values, or delete a line to leave that variable undefined. Blank lines and
|
|
129
|
+
`#` comments are ignored. On Windows the same variables are honored, falling
|
|
130
|
+
back to `notepad`. With `-e`/`--edit` the buffer also shows the `@include`
|
|
131
|
+
tree. Undefined variables are always reported on stderr; pass `--no-edit` to
|
|
132
|
+
silence everything and pass undefined names through untouched.
|
|
133
|
+
|
|
134
|
+
## Development
|
|
135
|
+
|
|
136
|
+
```console
|
|
137
|
+
python -m venv .venv
|
|
138
|
+
. .venv/bin/activate
|
|
139
|
+
pip install -e ".[dev]"
|
|
140
|
+
pytest
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
## License
|
|
144
|
+
|
|
145
|
+
MIT -- see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
sqlinclude/__init__.py,sha256=g04bR85Fwrb70J_5yvLatOM-vAZgsMOfAPNZ8NXsOIo,185
|
|
2
|
+
sqlinclude/__main__.py,sha256=CUYYxHB7ynJKm1CusZWozmjzBP8Na-xkN6Trbof1FIc,132
|
|
3
|
+
sqlinclude/cli.py,sha256=tuh18LCIQQhDPTAfIM2RoxTgzrkgCtNaNg9LhdavvPQ,7090
|
|
4
|
+
sqlinclude/editor.py,sha256=L9UgtFr6z3FB5f98D6YmC81Ynr9I2co38Mz5H-EVW8A,5645
|
|
5
|
+
sqlinclude/preprocess.py,sha256=hND8xXL1wsdz19Mvgon8I8tm61gw5SlFtFRBcsi0ht8,6566
|
|
6
|
+
sqlinclude-0.1.0.dist-info/licenses/LICENSE,sha256=dxVozrE5KYCaDnnUYIJ-pGTwZ9h_W8BN-MTWpM9N_-w,1068
|
|
7
|
+
sqlinclude-0.1.0.dist-info/METADATA,sha256=pvxO43eWpoePeHf_HYQSY_6UP-wiaIhl63kzePg5ymU,4559
|
|
8
|
+
sqlinclude-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
9
|
+
sqlinclude-0.1.0.dist-info/entry_points.txt,sha256=CgEfnBZQVk7BrkQnYozQWUSlgpJ2mBG9XMvqt1nkzW8,51
|
|
10
|
+
sqlinclude-0.1.0.dist-info/top_level.txt,sha256=NV8cYeEBTrXNwJHgG5LyGNupfmF52m2rwff6bMOOBxg,11
|
|
11
|
+
sqlinclude-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 lukasburski
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
sqlinclude
|