sqllocks-shape-sqlserver 0.9.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- shape_sqlserver/__init__.py +28 -0
- shape_sqlserver/auth.py +130 -0
- shape_sqlserver/catalog.py +181 -0
- shape_sqlserver/command.py +118 -0
- shape_sqlserver/profiler.py +454 -0
- shape_sqlserver/source.py +195 -0
- shape_sqlserver/sql.py +320 -0
- shape_sqlserver/testing.py +786 -0
- sqllocks_shape_sqlserver-0.9.1.dist-info/METADATA +51 -0
- sqllocks_shape_sqlserver-0.9.1.dist-info/RECORD +14 -0
- sqllocks_shape_sqlserver-0.9.1.dist-info/WHEEL +5 -0
- sqllocks_shape_sqlserver-0.9.1.dist-info/entry_points.txt +5 -0
- sqllocks_shape_sqlserver-0.9.1.dist-info/licenses/LICENSE +21 -0
- sqllocks_shape_sqlserver-0.9.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,786 @@
|
|
|
1
|
+
"""Test doubles for code that talks to SQL Server through DB-API (``pyodbc``).
|
|
2
|
+
|
|
3
|
+
Standard library only, and importable on its own (the parity harness loads this file by path).
|
|
4
|
+
``FakeConnection`` answers the catalog queries of :mod:`shape_sqlserver.sql` and the
|
|
5
|
+
``SELECT TOP n`` / ``SELECT ... ORDER BY`` reads from in-memory tables, with the same row and
|
|
6
|
+
value types ``pyodbc`` returns, so profiling and reading run end to end with no server.
|
|
7
|
+
:func:`scenario` builds deterministic databases for tests and for the nightly emulator job
|
|
8
|
+
(:func:`ddl` and :func:`insert_rows` load the same data into a real SQL Server).
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import datetime as dt
|
|
14
|
+
import re
|
|
15
|
+
import uuid
|
|
16
|
+
import zlib
|
|
17
|
+
from collections import namedtuple
|
|
18
|
+
from collections.abc import Callable, Iterable, Sequence
|
|
19
|
+
from dataclasses import dataclass, field
|
|
20
|
+
from decimal import Decimal
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass(frozen=True, slots=True)
|
|
25
|
+
class FakeColumn:
|
|
26
|
+
name: str
|
|
27
|
+
type_name: str
|
|
28
|
+
nullable: bool = True
|
|
29
|
+
identity: bool = False
|
|
30
|
+
max_length: int = 0
|
|
31
|
+
precision: int = 0
|
|
32
|
+
scale: int = 0
|
|
33
|
+
# A user-defined alias type (``CREATE TYPE alias FROM type_name``): the catalog reports it
|
|
34
|
+
# as the column's type, and ``type_name`` as its base system type.
|
|
35
|
+
alias: str | None = None
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass(slots=True)
|
|
39
|
+
class FakeTable:
|
|
40
|
+
name: str
|
|
41
|
+
columns: list[FakeColumn]
|
|
42
|
+
rows: list[tuple[Any, ...]] = field(default_factory=list)
|
|
43
|
+
schema: str = "dbo"
|
|
44
|
+
primary_key: tuple[str, ...] = ()
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass(frozen=True, slots=True)
|
|
48
|
+
class FakeForeignKey:
|
|
49
|
+
name: str
|
|
50
|
+
child_table: str
|
|
51
|
+
child_columns: tuple[str, ...]
|
|
52
|
+
parent_table: str
|
|
53
|
+
parent_columns: tuple[str, ...]
|
|
54
|
+
schema: str = "dbo"
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
_TopRead = re.compile(
|
|
58
|
+
r"^\s*SELECT\s+(?:TOP\s+(?P<top>\d+)\s+)?(?P<cols>.+?)\s+FROM\s+"
|
|
59
|
+
r"\[(?P<schema>(?:[^\]]|\]\])+)\]\.\[(?P<table>(?:[^\]]|\]\])+)\]"
|
|
60
|
+
r"(?:\s+ORDER\s+BY\s+(?P<order>.+?))?\s*$",
|
|
61
|
+
re.IGNORECASE | re.DOTALL,
|
|
62
|
+
)
|
|
63
|
+
_SPREAD = re.compile(
|
|
64
|
+
r"^\s*\(CAST\(CHECKSUM\((?P<cols>[^)]*)\) AS bigint\) \* (?P<mul>\d+)\) % (?P<mod>\d+)",
|
|
65
|
+
re.IGNORECASE,
|
|
66
|
+
)
|
|
67
|
+
_Bracketed = re.compile(r"\[((?:[^\]]|\]\])+)\]")
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def checksum(values: tuple[Any, ...]) -> int:
|
|
71
|
+
"""A stand-in for T-SQL ``CHECKSUM(a, b, ...)``: a signed 32-bit hash of the values. It is
|
|
72
|
+
deterministic, but it is not the server's function (which returns an integer unchanged):
|
|
73
|
+
tests that need the server's own order run against a real server."""
|
|
74
|
+
h = zlib.crc32(repr(values).encode())
|
|
75
|
+
return h - (1 << 32) if h >= 1 << 31 else h
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def spread_hash(values: tuple[Any, ...], mul: int = 1327217885, mod: int = 2147483647) -> int:
|
|
79
|
+
"""``(CAST(CHECKSUM(values) AS bigint) * mul) % mod`` as T-SQL computes it: the remainder
|
|
80
|
+
takes the sign of the dividend."""
|
|
81
|
+
product = checksum(values) * mul
|
|
82
|
+
return product % mod if product >= 0 else -((-product) % mod)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _row_type(fields: str) -> Any:
|
|
86
|
+
"""A row class with attribute access, like ``pyodbc.Row``."""
|
|
87
|
+
return namedtuple("Row", fields)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _sort_key(position: int) -> Callable[[tuple[Any, ...]], tuple[bool, Any]]:
|
|
91
|
+
"""NULLs first, like SQL Server."""
|
|
92
|
+
return lambda row: (row[position] is not None, row[position])
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _names(text: str) -> list[str]:
|
|
96
|
+
return [m.replace("]]", "]") for m in _Bracketed.findall(text)]
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class FakeCursor:
|
|
100
|
+
def __init__(self, conn: FakeConnection) -> None:
|
|
101
|
+
self._conn = conn
|
|
102
|
+
self._rows: list[Any] = []
|
|
103
|
+
self._pos = 0
|
|
104
|
+
self.description: list[tuple[Any, ...]] | None = None
|
|
105
|
+
self.closed = False
|
|
106
|
+
|
|
107
|
+
def _set(self, names: Sequence[str], rows: Iterable[Any]) -> None:
|
|
108
|
+
self.description = [(n, None, None, None, None, None, None) for n in names]
|
|
109
|
+
self._rows = list(rows)
|
|
110
|
+
self._pos = 0
|
|
111
|
+
|
|
112
|
+
def execute(self, sql: str, params: Sequence[Any] = ()) -> FakeCursor:
|
|
113
|
+
if self.closed:
|
|
114
|
+
raise RuntimeError("cursor is closed")
|
|
115
|
+
self._conn.statements.append(sql.strip())
|
|
116
|
+
self._conn.dispatch(self, sql, tuple(params))
|
|
117
|
+
return self
|
|
118
|
+
|
|
119
|
+
def fetchall(self) -> list[Any]:
|
|
120
|
+
rows, self._pos = self._rows[self._pos :], len(self._rows)
|
|
121
|
+
return rows
|
|
122
|
+
|
|
123
|
+
def fetchmany(self, size: int = 1) -> list[Any]:
|
|
124
|
+
rows = self._rows[self._pos : self._pos + size]
|
|
125
|
+
self._pos += len(rows)
|
|
126
|
+
return rows
|
|
127
|
+
|
|
128
|
+
def close(self) -> None:
|
|
129
|
+
self.closed = True
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
class FakeConnection:
|
|
133
|
+
"""An in-memory SQL Server: ``tables``, declared ``foreign_keys``, and what to break.
|
|
134
|
+
|
|
135
|
+
``statements`` records every SQL text executed. ``fail_reads`` names tables whose data
|
|
136
|
+
queries raise (catalog queries still work).
|
|
137
|
+
"""
|
|
138
|
+
|
|
139
|
+
def __init__(
|
|
140
|
+
self,
|
|
141
|
+
tables: Sequence[FakeTable],
|
|
142
|
+
foreign_keys: Sequence[FakeForeignKey] = (),
|
|
143
|
+
*,
|
|
144
|
+
fail_reads: Iterable[str] = (),
|
|
145
|
+
row_counts: dict[str, int] | None = None,
|
|
146
|
+
) -> None:
|
|
147
|
+
self.tables = list(tables)
|
|
148
|
+
self.foreign_keys = list(foreign_keys)
|
|
149
|
+
self.fail_reads = set(fail_reads)
|
|
150
|
+
self.row_counts = row_counts or {}
|
|
151
|
+
self.statements: list[str] = []
|
|
152
|
+
self.closed = False
|
|
153
|
+
|
|
154
|
+
def cursor(self) -> FakeCursor:
|
|
155
|
+
if self.closed:
|
|
156
|
+
raise RuntimeError("connection is closed")
|
|
157
|
+
return FakeCursor(self)
|
|
158
|
+
|
|
159
|
+
def close(self) -> None:
|
|
160
|
+
self.closed = True
|
|
161
|
+
|
|
162
|
+
def _object_id(self, table: FakeTable) -> int:
|
|
163
|
+
return 1000 + self.tables.index(table)
|
|
164
|
+
|
|
165
|
+
def _by_id(self, object_id: int) -> FakeTable:
|
|
166
|
+
return self.tables[int(object_id) - 1000]
|
|
167
|
+
|
|
168
|
+
def dispatch(self, cur: FakeCursor, sql: str, params: tuple[Any, ...]) -> None:
|
|
169
|
+
text = " ".join(sql.split())
|
|
170
|
+
if "FROM sys.tables t JOIN sys.schemas s" in text and "sys.partitions" in text:
|
|
171
|
+
rows_t = _row_type("schema_name table_name row_count")
|
|
172
|
+
cur._set(
|
|
173
|
+
rows_t._fields,
|
|
174
|
+
[
|
|
175
|
+
rows_t(
|
|
176
|
+
t.schema,
|
|
177
|
+
t.name,
|
|
178
|
+
self.row_counts.get(t.name, len(t.rows)),
|
|
179
|
+
)
|
|
180
|
+
for t in sorted(self.tables, key=lambda x: x.name)
|
|
181
|
+
if t.schema == params[0]
|
|
182
|
+
],
|
|
183
|
+
)
|
|
184
|
+
elif text.startswith("SELECT s.name AS schema_name, t.name AS table_name, t.object_id"):
|
|
185
|
+
rows_t = _row_type("schema_name table_name object_id")
|
|
186
|
+
cur._set(
|
|
187
|
+
rows_t._fields,
|
|
188
|
+
[
|
|
189
|
+
rows_t(t.schema, t.name, self._object_id(t))
|
|
190
|
+
for t in sorted(self.tables, key=lambda x: x.name)
|
|
191
|
+
if t.schema == params[0]
|
|
192
|
+
],
|
|
193
|
+
)
|
|
194
|
+
elif "FROM sys.columns c JOIN sys.types tp" in text:
|
|
195
|
+
rows_t = _row_type(
|
|
196
|
+
"column_name type_name base_type_name max_length precision scale "
|
|
197
|
+
"is_nullable is_identity column_id",
|
|
198
|
+
)
|
|
199
|
+
table = self._by_id(params[0])
|
|
200
|
+
cur._set(
|
|
201
|
+
rows_t._fields,
|
|
202
|
+
[
|
|
203
|
+
rows_t(
|
|
204
|
+
c.name,
|
|
205
|
+
c.alias or c.type_name,
|
|
206
|
+
c.type_name,
|
|
207
|
+
c.max_length,
|
|
208
|
+
c.precision,
|
|
209
|
+
c.scale,
|
|
210
|
+
c.nullable,
|
|
211
|
+
c.identity,
|
|
212
|
+
i + 1,
|
|
213
|
+
)
|
|
214
|
+
for i, c in enumerate(table.columns)
|
|
215
|
+
],
|
|
216
|
+
)
|
|
217
|
+
elif "FROM sys.key_constraints kc" in text:
|
|
218
|
+
rows_t = _row_type("column_name")
|
|
219
|
+
table = self._by_id(params[0])
|
|
220
|
+
cur._set(rows_t._fields, [rows_t(c) for c in table.primary_key])
|
|
221
|
+
elif "FROM sys.foreign_keys fk" in text:
|
|
222
|
+
rows_t = _row_type(
|
|
223
|
+
"fk_name child_table child_column parent_table parent_column ordinal"
|
|
224
|
+
)
|
|
225
|
+
out = []
|
|
226
|
+
for fk in sorted(self.foreign_keys, key=lambda f: f.name):
|
|
227
|
+
if fk.schema != params[0]:
|
|
228
|
+
continue
|
|
229
|
+
for i, (cc, pc) in enumerate(zip(fk.child_columns, fk.parent_columns, strict=True)):
|
|
230
|
+
out.append(rows_t(fk.name, fk.child_table, cc, fk.parent_table, pc, i + 1))
|
|
231
|
+
cur._set(rows_t._fields, out)
|
|
232
|
+
else:
|
|
233
|
+
self._read(cur, text)
|
|
234
|
+
|
|
235
|
+
def _read(self, cur: FakeCursor, text: str) -> None:
|
|
236
|
+
m = _TopRead.match(text)
|
|
237
|
+
if not m:
|
|
238
|
+
raise RuntimeError(f"FakeConnection cannot run: {text}")
|
|
239
|
+
name = m["table"].replace("]]", "]")
|
|
240
|
+
table = next((t for t in self.tables if t.name == name and t.schema == m["schema"]), None)
|
|
241
|
+
if table is None:
|
|
242
|
+
raise RuntimeError(f"Invalid object name '{m['schema']}.{name}'.")
|
|
243
|
+
if name in self.fail_reads:
|
|
244
|
+
raise RuntimeError(f"The SELECT permission was denied on the object '{name}'.")
|
|
245
|
+
index = {c.name: i for i, c in enumerate(table.columns)}
|
|
246
|
+
wanted = list(index) if m["cols"].strip() == "*" else _names(m["cols"])
|
|
247
|
+
rows = list(table.rows)
|
|
248
|
+
if m["order"]:
|
|
249
|
+
order = m["order"]
|
|
250
|
+
keys = _names(order)
|
|
251
|
+
hashed = _SPREAD.match(order)
|
|
252
|
+
if hashed: # ORDER BY (CAST(CHECKSUM(a, b) AS bigint) * m) % p, c: hash, then columns
|
|
253
|
+
keys = keys[len(_names(hashed["cols"])) :]
|
|
254
|
+
for key in reversed(keys):
|
|
255
|
+
rows.sort(key=_sort_key(index[key]))
|
|
256
|
+
if hashed:
|
|
257
|
+
cols = [index[c] for c in _names(hashed["cols"])]
|
|
258
|
+
mul, mod = int(hashed["mul"]), int(hashed["mod"])
|
|
259
|
+
rows.sort(key=lambda r: spread_hash(tuple(r[i] for i in cols), mul, mod))
|
|
260
|
+
if m["top"]:
|
|
261
|
+
rows = rows[: int(m["top"])]
|
|
262
|
+
pick = [index[c] for c in wanted]
|
|
263
|
+
cur._set(wanted, [tuple(r[i] for i in pick) for r in rows])
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
# --- deterministic data -------------------------------------------------------------------
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
class Rng:
|
|
270
|
+
"""A tiny splitmix64 generator, identical on every Python version and platform."""
|
|
271
|
+
|
|
272
|
+
def __init__(self, seed: int) -> None:
|
|
273
|
+
self._s = seed & 0xFFFFFFFFFFFFFFFF
|
|
274
|
+
|
|
275
|
+
def u64(self) -> int:
|
|
276
|
+
self._s = (self._s + 0x9E3779B97F4A7C15) & 0xFFFFFFFFFFFFFFFF
|
|
277
|
+
z = self._s
|
|
278
|
+
z = ((z ^ (z >> 30)) * 0xBF58476D1CE4E5B9) & 0xFFFFFFFFFFFFFFFF
|
|
279
|
+
z = ((z ^ (z >> 27)) * 0x94D049BB133111EB) & 0xFFFFFFFFFFFFFFFF
|
|
280
|
+
return z ^ (z >> 31)
|
|
281
|
+
|
|
282
|
+
def below(self, n: int) -> int:
|
|
283
|
+
return self.u64() % n
|
|
284
|
+
|
|
285
|
+
def unit(self) -> float:
|
|
286
|
+
return (self.u64() >> 11) / float(1 << 53)
|
|
287
|
+
|
|
288
|
+
def pick(self, items: Sequence[Any], weights: Sequence[int] | None = None) -> Any:
|
|
289
|
+
if weights is None:
|
|
290
|
+
return items[self.below(len(items))]
|
|
291
|
+
point = self.below(sum(weights))
|
|
292
|
+
for item, w in zip(items, weights, strict=True):
|
|
293
|
+
if point < w:
|
|
294
|
+
return item
|
|
295
|
+
point -= w
|
|
296
|
+
return items[-1]
|
|
297
|
+
|
|
298
|
+
def maybe_null(self, value: Any, percent: int) -> Any:
|
|
299
|
+
return None if self.below(100) < percent else value
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def _money(cents: int) -> Decimal:
|
|
303
|
+
return Decimal(f"{cents // 100}.{cents % 100:02d}")
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
_BASE = dt.datetime(2020, 1, 1, 8, 0, 0)
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def _col(name: str, type_name: str, **kw: Any) -> FakeColumn:
|
|
310
|
+
return FakeColumn(name, type_name, **kw)
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def _retail(scale: int, rng: Rng) -> tuple[list[FakeTable], list[FakeForeignKey]]:
|
|
314
|
+
n_cust, n_prod, n_ord, n_item = 2500 * scale, 300 * scale, 4000 * scale, 6000 * scale
|
|
315
|
+
segments = ["consumer", "corporate", "home office", "small business"]
|
|
316
|
+
categories = [f"category {i}" for i in range(9)]
|
|
317
|
+
statuses = ["new", "paid", "shipped", "delivered", "returned"]
|
|
318
|
+
customers = []
|
|
319
|
+
for i in range(1, n_cust + 1):
|
|
320
|
+
customers.append(
|
|
321
|
+
(
|
|
322
|
+
i,
|
|
323
|
+
f"Customer {i}",
|
|
324
|
+
f"user{i}@example.test",
|
|
325
|
+
rng.pick(segments, [50, 25, 15, 10]),
|
|
326
|
+
rng.maybe_null(_money(rng.below(900000)), 8),
|
|
327
|
+
int(300 + rng.below(550)),
|
|
328
|
+
_BASE + dt.timedelta(minutes=rng.below(500000)),
|
|
329
|
+
rng.maybe_null(dt.date(1950, 1, 1) + dt.timedelta(days=rng.below(20000)), 5),
|
|
330
|
+
rng.maybe_null(rng.below(100) < 60, 2),
|
|
331
|
+
rng.maybe_null(int(rng.unit() ** 3 * 5000), 10),
|
|
332
|
+
f"R{rng.below(15):02d}",
|
|
333
|
+
str(uuid.UUID(int=rng.u64() << 64 | rng.u64())),
|
|
334
|
+
)
|
|
335
|
+
)
|
|
336
|
+
products = []
|
|
337
|
+
for i in range(1, n_prod + 1):
|
|
338
|
+
products.append(
|
|
339
|
+
(
|
|
340
|
+
i,
|
|
341
|
+
f"SKU-{i:05d}",
|
|
342
|
+
rng.pick(categories),
|
|
343
|
+
_money(100 + rng.below(50000)),
|
|
344
|
+
rng.maybe_null(round(0.1 + rng.unit() * 30, 3), 15),
|
|
345
|
+
round(1 + rng.unit() * 4, 1),
|
|
346
|
+
rng.below(255),
|
|
347
|
+
rng.maybe_null(f"Product {i} " + "x" * rng.below(40), 20),
|
|
348
|
+
)
|
|
349
|
+
)
|
|
350
|
+
orders = []
|
|
351
|
+
for i in range(1, n_ord + 1):
|
|
352
|
+
orders.append(
|
|
353
|
+
(
|
|
354
|
+
10_000_000_000 + i,
|
|
355
|
+
1 + rng.below(n_cust),
|
|
356
|
+
dt.date(2022, 1, 1) + dt.timedelta(days=rng.below(900)),
|
|
357
|
+
rng.pick(statuses, [10, 30, 25, 30, 5]),
|
|
358
|
+
_money(500 + rng.below(300000)),
|
|
359
|
+
rng.maybe_null(dt.time(rng.below(24), rng.below(60), rng.below(60)), 30),
|
|
360
|
+
rng.maybe_null(f"note {rng.below(200)}", 70),
|
|
361
|
+
)
|
|
362
|
+
)
|
|
363
|
+
items = []
|
|
364
|
+
line = 0
|
|
365
|
+
for i in range(n_item):
|
|
366
|
+
if i % 3 == 0:
|
|
367
|
+
line = 0
|
|
368
|
+
line += 1
|
|
369
|
+
items.append(
|
|
370
|
+
(
|
|
371
|
+
10_000_000_000 + 1 + (i // 3) % n_ord,
|
|
372
|
+
line,
|
|
373
|
+
1 + rng.below(n_prod),
|
|
374
|
+
1 + rng.below(9),
|
|
375
|
+
_money(100 + rng.below(50000)),
|
|
376
|
+
rng.maybe_null(round(rng.unit() * 0.3, 2), 60),
|
|
377
|
+
)
|
|
378
|
+
)
|
|
379
|
+
tables = [
|
|
380
|
+
FakeTable(
|
|
381
|
+
"customer",
|
|
382
|
+
[
|
|
383
|
+
_col(
|
|
384
|
+
"customer_id", "int", nullable=False, identity=True, max_length=4, precision=10
|
|
385
|
+
),
|
|
386
|
+
_col("name", "nvarchar", max_length=160),
|
|
387
|
+
_col("email", "varchar", max_length=120),
|
|
388
|
+
_col("segment", "varchar", max_length=20),
|
|
389
|
+
_col("balance", "decimal", max_length=9, precision=12, scale=2),
|
|
390
|
+
_col("credit_score", "smallint", max_length=2, precision=5),
|
|
391
|
+
_col("created_at", "datetime2", max_length=8, precision=27, scale=7),
|
|
392
|
+
_col("birth_date", "date", max_length=3, precision=10),
|
|
393
|
+
_col("is_active", "bit", max_length=1, precision=1),
|
|
394
|
+
_col("loyalty_points", "int", max_length=4, precision=10),
|
|
395
|
+
_col("region_code", "char", max_length=3),
|
|
396
|
+
_col("guid", "uniqueidentifier", max_length=16),
|
|
397
|
+
],
|
|
398
|
+
customers,
|
|
399
|
+
primary_key=("customer_id",),
|
|
400
|
+
),
|
|
401
|
+
FakeTable(
|
|
402
|
+
"product",
|
|
403
|
+
[
|
|
404
|
+
_col("product_id", "int", nullable=False, max_length=4, precision=10),
|
|
405
|
+
_col("sku", "varchar", max_length=20),
|
|
406
|
+
_col("category", "varchar", max_length=30),
|
|
407
|
+
_col("price", "money", max_length=8, precision=19, scale=4),
|
|
408
|
+
_col("weight_kg", "float", max_length=8, precision=53),
|
|
409
|
+
_col("rating", "real", max_length=4, precision=24),
|
|
410
|
+
_col("in_stock", "tinyint", max_length=1, precision=3),
|
|
411
|
+
_col("description", "nvarchar", max_length=-1),
|
|
412
|
+
],
|
|
413
|
+
products,
|
|
414
|
+
primary_key=("product_id",),
|
|
415
|
+
),
|
|
416
|
+
FakeTable(
|
|
417
|
+
"orders",
|
|
418
|
+
[
|
|
419
|
+
_col("order_id", "bigint", nullable=False, max_length=8, precision=19),
|
|
420
|
+
_col("customer_id", "int", nullable=False, max_length=4, precision=10),
|
|
421
|
+
_col("order_date", "date", max_length=3, precision=10),
|
|
422
|
+
_col("status", "varchar", max_length=12),
|
|
423
|
+
_col("total", "decimal", max_length=9, precision=10, scale=2),
|
|
424
|
+
_col("ship_time", "time", max_length=5, precision=16, scale=7),
|
|
425
|
+
_col("notes", "varchar", max_length=200),
|
|
426
|
+
],
|
|
427
|
+
orders,
|
|
428
|
+
primary_key=("order_id",),
|
|
429
|
+
),
|
|
430
|
+
FakeTable(
|
|
431
|
+
"order_item",
|
|
432
|
+
[
|
|
433
|
+
_col("order_id", "bigint", nullable=False, max_length=8, precision=19),
|
|
434
|
+
_col("line_no", "int", nullable=False, max_length=4, precision=10),
|
|
435
|
+
_col("product_id", "int", nullable=False, max_length=4, precision=10),
|
|
436
|
+
_col("quantity", "int", max_length=4, precision=10),
|
|
437
|
+
_col("unit_price", "decimal", max_length=9, precision=10, scale=2),
|
|
438
|
+
_col("discount", "float", max_length=8, precision=53),
|
|
439
|
+
],
|
|
440
|
+
items,
|
|
441
|
+
primary_key=("order_id", "line_no"),
|
|
442
|
+
),
|
|
443
|
+
FakeTable(
|
|
444
|
+
"audit_log",
|
|
445
|
+
[
|
|
446
|
+
_col("id", "int", nullable=False, identity=True, max_length=4, precision=10),
|
|
447
|
+
_col("event", "varchar", max_length=50),
|
|
448
|
+
_col("at", "datetime2", max_length=8, precision=27, scale=7),
|
|
449
|
+
],
|
|
450
|
+
[],
|
|
451
|
+
primary_key=("id",),
|
|
452
|
+
),
|
|
453
|
+
FakeTable(
|
|
454
|
+
"config",
|
|
455
|
+
[_col("name", "varchar", nullable=False, max_length=30), _col("value", "int")],
|
|
456
|
+
[("version", 3)],
|
|
457
|
+
primary_key=("name",),
|
|
458
|
+
),
|
|
459
|
+
]
|
|
460
|
+
fks = [
|
|
461
|
+
FakeForeignKey(
|
|
462
|
+
"fk_orders_customer", "orders", ("customer_id",), "customer", ("customer_id",)
|
|
463
|
+
),
|
|
464
|
+
FakeForeignKey("fk_item_orders", "order_item", ("order_id",), "orders", ("order_id",)),
|
|
465
|
+
FakeForeignKey(
|
|
466
|
+
"fk_item_product", "order_item", ("product_id",), "product", ("product_id",)
|
|
467
|
+
),
|
|
468
|
+
]
|
|
469
|
+
return tables, fks
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def _warehouse(scale: int, rng: Rng, *, empty: bool, contained: bool) -> list[FakeTable]:
|
|
473
|
+
"""Tables with no declared keys, as a Fabric warehouse has. ``contained`` makes the fact
|
|
474
|
+
table's key columns hold only dimension keys; otherwise they hold unrelated numbers."""
|
|
475
|
+
n_c, n_p, n_f = (0, 0, 0) if empty else (120 * scale, 40 * scale, 900 * scale)
|
|
476
|
+
regions = ["north", "south", "east", "west"]
|
|
477
|
+
dimc = [(i, f"Name {i}", rng.pick(regions)) for i in range(1, n_c + 1)]
|
|
478
|
+
dimp = [(i, f"Widget {i}", f"line {i % 5}") for i in range(1, n_p + 1)]
|
|
479
|
+
fact = []
|
|
480
|
+
for i in range(1, n_f + 1):
|
|
481
|
+
c = 1 + rng.below(n_c) if contained else 900_000 + rng.below(50)
|
|
482
|
+
p = 1 + rng.below(n_p) if contained else 800_000 + rng.below(30)
|
|
483
|
+
fact.append(
|
|
484
|
+
(
|
|
485
|
+
i,
|
|
486
|
+
c,
|
|
487
|
+
p,
|
|
488
|
+
_money(100 + rng.below(90000)),
|
|
489
|
+
dt.date(2024, 1, 1) + dt.timedelta(days=rng.below(300)),
|
|
490
|
+
)
|
|
491
|
+
)
|
|
492
|
+
return [
|
|
493
|
+
FakeTable(
|
|
494
|
+
"dimcustomer",
|
|
495
|
+
[
|
|
496
|
+
_col("customer_key", "int", nullable=False, max_length=4, precision=10),
|
|
497
|
+
_col("customer_name", "varchar", max_length=100),
|
|
498
|
+
_col("region", "varchar", max_length=20),
|
|
499
|
+
],
|
|
500
|
+
dimc,
|
|
501
|
+
),
|
|
502
|
+
FakeTable(
|
|
503
|
+
"dimproduct",
|
|
504
|
+
[
|
|
505
|
+
_col("product_key", "int", nullable=False, max_length=4, precision=10),
|
|
506
|
+
_col("product_name", "varchar", max_length=100),
|
|
507
|
+
_col("product_line", "varchar", max_length=20),
|
|
508
|
+
],
|
|
509
|
+
dimp,
|
|
510
|
+
),
|
|
511
|
+
FakeTable(
|
|
512
|
+
"factsales",
|
|
513
|
+
[
|
|
514
|
+
_col("sale_id", "int", nullable=False, max_length=4, precision=10),
|
|
515
|
+
_col("customer_key", "int", max_length=4, precision=10),
|
|
516
|
+
_col("product_key", "int", max_length=4, precision=10),
|
|
517
|
+
_col("amount", "decimal", max_length=9, precision=12, scale=2),
|
|
518
|
+
_col("sale_date", "date", max_length=3, precision=10),
|
|
519
|
+
],
|
|
520
|
+
fact,
|
|
521
|
+
),
|
|
522
|
+
]
|
|
523
|
+
|
|
524
|
+
|
|
525
|
+
def _name_only(scale: int, rng: Rng) -> list[FakeTable]:
|
|
526
|
+
"""No keys, and data that does not show a relationship; only the names do."""
|
|
527
|
+
n_c, n_o = 60 * scale, 200 * scale
|
|
528
|
+
customer = [(i, f"c{i}") for i in range(1, n_c + 1)]
|
|
529
|
+
orders = [(500_000 + i, 700_000 + rng.below(40), _money(rng.below(9000))) for i in range(n_o)]
|
|
530
|
+
return [
|
|
531
|
+
FakeTable(
|
|
532
|
+
"customer",
|
|
533
|
+
[
|
|
534
|
+
_col("id", "int", nullable=False, precision=10),
|
|
535
|
+
_col("label", "varchar", max_length=10),
|
|
536
|
+
],
|
|
537
|
+
customer,
|
|
538
|
+
),
|
|
539
|
+
FakeTable(
|
|
540
|
+
"orders",
|
|
541
|
+
[
|
|
542
|
+
_col("order_id", "int", nullable=False, precision=10),
|
|
543
|
+
_col("customer_id", "int", precision=10),
|
|
544
|
+
_col("amount", "decimal", precision=10, scale=2),
|
|
545
|
+
],
|
|
546
|
+
orders,
|
|
547
|
+
),
|
|
548
|
+
]
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
def _id_named(scale: int, rng: Rng) -> list[FakeTable]:
|
|
552
|
+
"""No declared keys, ``*_id`` columns whose values all exist in the parent table."""
|
|
553
|
+
n_c, n_o = 80 * scale, 300 * scale
|
|
554
|
+
customer = [(i, f"c{i}") for i in range(1, n_c + 1)]
|
|
555
|
+
orders = [(i, 1 + rng.below(n_c), _money(rng.below(9000))) for i in range(1, n_o + 1)]
|
|
556
|
+
return [
|
|
557
|
+
FakeTable(
|
|
558
|
+
"customer",
|
|
559
|
+
[
|
|
560
|
+
_col("customer_id", "int", nullable=False, precision=10),
|
|
561
|
+
_col("label", "varchar", max_length=10),
|
|
562
|
+
],
|
|
563
|
+
customer,
|
|
564
|
+
),
|
|
565
|
+
FakeTable(
|
|
566
|
+
"orders",
|
|
567
|
+
[
|
|
568
|
+
_col("order_id", "int", nullable=False, precision=10),
|
|
569
|
+
_col("customer_id", "int", precision=10),
|
|
570
|
+
_col("amount", "decimal", precision=10, scale=2),
|
|
571
|
+
],
|
|
572
|
+
orders,
|
|
573
|
+
),
|
|
574
|
+
]
|
|
575
|
+
|
|
576
|
+
|
|
577
|
+
def _mixed_keys(scale: int, rng: Rng) -> tuple[list[FakeTable], list[FakeForeignKey]]:
|
|
578
|
+
"""One declared foreign key (``orders.customer_id``) and two undeclared ones beside it:
|
|
579
|
+
``orders.product_id`` (every value exists in ``product``: the data shows it) and
|
|
580
|
+
``orders.region_key`` (numbers no region has; only the name suggests it)."""
|
|
581
|
+
n_r, n_c, n_p, n_o = 8, 60 * scale, 25 * scale, 240 * scale
|
|
582
|
+
region = [(i, f"region {i}") for i in range(1, n_r + 1)]
|
|
583
|
+
customer = [(i, f"c{i}") for i in range(1, n_c + 1)]
|
|
584
|
+
product = [(i, f"p{i}") for i in range(1, n_p + 1)]
|
|
585
|
+
orders = [
|
|
586
|
+
(i, 1 + rng.below(n_c), 1 + rng.below(n_p), 700 + rng.below(9)) for i in range(1, n_o + 1)
|
|
587
|
+
]
|
|
588
|
+
tables = [
|
|
589
|
+
FakeTable(
|
|
590
|
+
"region",
|
|
591
|
+
[
|
|
592
|
+
_col("region_id", "int", nullable=False, precision=10),
|
|
593
|
+
_col("label", "varchar", max_length=20),
|
|
594
|
+
],
|
|
595
|
+
region,
|
|
596
|
+
primary_key=("region_id",),
|
|
597
|
+
),
|
|
598
|
+
FakeTable(
|
|
599
|
+
"customer",
|
|
600
|
+
[
|
|
601
|
+
_col("customer_id", "int", nullable=False, precision=10),
|
|
602
|
+
_col("label", "varchar", max_length=20),
|
|
603
|
+
],
|
|
604
|
+
customer,
|
|
605
|
+
primary_key=("customer_id",),
|
|
606
|
+
),
|
|
607
|
+
FakeTable(
|
|
608
|
+
"product",
|
|
609
|
+
[
|
|
610
|
+
_col("product_id", "int", nullable=False, precision=10),
|
|
611
|
+
_col("label", "varchar", max_length=20),
|
|
612
|
+
],
|
|
613
|
+
product,
|
|
614
|
+
primary_key=("product_id",),
|
|
615
|
+
),
|
|
616
|
+
FakeTable(
|
|
617
|
+
"orders",
|
|
618
|
+
[
|
|
619
|
+
_col("order_id", "int", nullable=False, precision=10),
|
|
620
|
+
_col("customer_id", "int", nullable=False, precision=10),
|
|
621
|
+
_col("product_id", "int", precision=10),
|
|
622
|
+
_col("region_key", "int", precision=10),
|
|
623
|
+
],
|
|
624
|
+
orders,
|
|
625
|
+
primary_key=("order_id",),
|
|
626
|
+
),
|
|
627
|
+
]
|
|
628
|
+
fks = [
|
|
629
|
+
FakeForeignKey(
|
|
630
|
+
"fk_orders_customer", "orders", ("customer_id",), "customer", ("customer_id",)
|
|
631
|
+
)
|
|
632
|
+
]
|
|
633
|
+
return tables, fks
|
|
634
|
+
|
|
635
|
+
|
|
636
|
+
def _key_names(scale: int, rng: Rng) -> list[FakeTable]:
|
|
637
|
+
"""No declared keys and data that shows no relationship (only names do). ``invoices.valid``
|
|
638
|
+
and ``invoices.paid`` end in ``id`` but are not ids (tables ``val`` and ``pa`` exist to be
|
|
639
|
+
matched wrongly); ``invoices.customer_id`` comes first but is a foreign key, so the key of
|
|
640
|
+
``invoices`` is ``invoice_id``; ``notes`` has only a foreign key and a text column, so it
|
|
641
|
+
has no key at all."""
|
|
642
|
+
n_c, n_i, n_n = 40 * scale, 150 * scale, 30 * scale
|
|
643
|
+
customer = [(i, f"c{i}") for i in range(1, n_c + 1)]
|
|
644
|
+
invoices = [
|
|
645
|
+
(9000 + rng.below(n_c), i, rng.below(2), rng.below(2), f"inv {i}")
|
|
646
|
+
for i in range(1, n_i + 1)
|
|
647
|
+
]
|
|
648
|
+
notes = [(f"note {i % 7}", 9000 + rng.below(n_c)) for i in range(n_n)]
|
|
649
|
+
flags = [(0, "no"), (1, "yes")]
|
|
650
|
+
return [
|
|
651
|
+
FakeTable(
|
|
652
|
+
"customer",
|
|
653
|
+
[
|
|
654
|
+
_col("customer_id", "int", nullable=False, precision=10),
|
|
655
|
+
_col("label", "varchar", max_length=20),
|
|
656
|
+
],
|
|
657
|
+
customer,
|
|
658
|
+
),
|
|
659
|
+
FakeTable(
|
|
660
|
+
"invoices",
|
|
661
|
+
[
|
|
662
|
+
_col("customer_id", "int", precision=10),
|
|
663
|
+
_col("invoice_id", "int", nullable=False, precision=10),
|
|
664
|
+
_col("valid", "int", precision=10),
|
|
665
|
+
_col("paid", "int", precision=10),
|
|
666
|
+
_col("memo", "varchar", max_length=20),
|
|
667
|
+
],
|
|
668
|
+
invoices,
|
|
669
|
+
),
|
|
670
|
+
FakeTable(
|
|
671
|
+
"notes",
|
|
672
|
+
[_col("note", "varchar", max_length=20), _col("customer_id", "int", precision=10)],
|
|
673
|
+
notes,
|
|
674
|
+
),
|
|
675
|
+
FakeTable(
|
|
676
|
+
"pa",
|
|
677
|
+
[
|
|
678
|
+
_col("id", "int", nullable=False, precision=10),
|
|
679
|
+
_col("label", "varchar", max_length=5),
|
|
680
|
+
],
|
|
681
|
+
flags,
|
|
682
|
+
),
|
|
683
|
+
FakeTable(
|
|
684
|
+
"val",
|
|
685
|
+
[
|
|
686
|
+
_col("id", "int", nullable=False, precision=10),
|
|
687
|
+
_col("label", "varchar", max_length=5),
|
|
688
|
+
],
|
|
689
|
+
flags,
|
|
690
|
+
),
|
|
691
|
+
]
|
|
692
|
+
|
|
693
|
+
|
|
694
|
+
def _no_keys(
|
|
695
|
+
build: Callable[[int, Rng], list[FakeTable]],
|
|
696
|
+
) -> Callable[[int, Rng], tuple[list[FakeTable], list[FakeForeignKey]]]:
|
|
697
|
+
def make(scale: int, rng: Rng) -> tuple[list[FakeTable], list[FakeForeignKey]]:
|
|
698
|
+
return build(scale, rng), []
|
|
699
|
+
|
|
700
|
+
return make
|
|
701
|
+
|
|
702
|
+
|
|
703
|
+
def _warehouse_full(scale: int, rng: Rng) -> list[FakeTable]:
|
|
704
|
+
return _warehouse(scale, rng, empty=False, contained=True)
|
|
705
|
+
|
|
706
|
+
|
|
707
|
+
def _warehouse_empty(scale: int, rng: Rng) -> list[FakeTable]:
|
|
708
|
+
return _warehouse(scale, rng, empty=True, contained=True)
|
|
709
|
+
|
|
710
|
+
|
|
711
|
+
SCENARIOS: dict[str, Callable[[int, Rng], tuple[list[FakeTable], list[FakeForeignKey]]]] = {
|
|
712
|
+
"retail": _retail,
|
|
713
|
+
"warehouse": _no_keys(_warehouse_full),
|
|
714
|
+
"warehouse_empty": _no_keys(_warehouse_empty),
|
|
715
|
+
"name_only": _no_keys(_name_only),
|
|
716
|
+
"id_named": _no_keys(_id_named),
|
|
717
|
+
"mixed_keys": _mixed_keys,
|
|
718
|
+
"key_names": _no_keys(_key_names),
|
|
719
|
+
}
|
|
720
|
+
|
|
721
|
+
|
|
722
|
+
def scenario(name: str, scale: int = 1, seed: int = 7) -> FakeConnection:
|
|
723
|
+
"""A deterministic database: ``retail`` (declared keys, tables larger than a 1000-row
|
|
724
|
+
sample, an empty and a one-row table), ``warehouse`` (no keys; key columns named ``*_key``,
|
|
725
|
+
linked by name), ``warehouse_empty`` (no keys, no rows) or ``name_only`` (no keys; only
|
|
726
|
+
column names suggest a relationship) or ``id_named`` (no keys; ``*_id`` columns whose
|
|
727
|
+
values all exist in the parent table), ``mixed_keys`` (one declared foreign key, and
|
|
728
|
+
undeclared ones beside it) or ``key_names`` (no keys; ``paid``/``valid`` columns and tables
|
|
729
|
+
named like their stems, a foreign key first in its table, a table with no key column)."""
|
|
730
|
+
tables, fks = SCENARIOS[name](scale, Rng(seed))
|
|
731
|
+
return FakeConnection(tables, fks)
|
|
732
|
+
|
|
733
|
+
|
|
734
|
+
# --- loading the same data into a real server --------------------------------------------
|
|
735
|
+
|
|
736
|
+
|
|
737
|
+
def _column_ddl(c: FakeColumn) -> str:
|
|
738
|
+
t = c.type_name
|
|
739
|
+
if t in ("varchar", "nvarchar", "char", "nchar", "varbinary", "binary"):
|
|
740
|
+
size = (
|
|
741
|
+
"max"
|
|
742
|
+
if c.max_length == -1
|
|
743
|
+
else str(c.max_length // 2 if t.startswith("n") else c.max_length)
|
|
744
|
+
)
|
|
745
|
+
t = f"{t}({size})"
|
|
746
|
+
elif t in ("decimal", "numeric"):
|
|
747
|
+
t = f"{t}({c.precision},{c.scale})"
|
|
748
|
+
elif t in ("datetime2", "time"):
|
|
749
|
+
t = f"{t}({c.scale or 7})"
|
|
750
|
+
ident = " IDENTITY(1,1)" if c.identity else ""
|
|
751
|
+
return f"[{c.name}] {t}{ident} {'NULL' if c.nullable else 'NOT NULL'}"
|
|
752
|
+
|
|
753
|
+
|
|
754
|
+
def ddl(conn: FakeConnection, schema: str = "dbo") -> list[str]:
|
|
755
|
+
"""``CREATE TABLE`` / ``ALTER TABLE ... FOREIGN KEY`` statements for ``conn``'s tables."""
|
|
756
|
+
out = []
|
|
757
|
+
for t in conn.tables:
|
|
758
|
+
cols = [_column_ddl(c) for c in t.columns]
|
|
759
|
+
if t.primary_key:
|
|
760
|
+
cols.append(f"PRIMARY KEY ({', '.join(f'[{c}]' for c in t.primary_key)})")
|
|
761
|
+
out.append(f"CREATE TABLE [{schema}].[{t.name}] ({', '.join(cols)})")
|
|
762
|
+
for fk in conn.foreign_keys:
|
|
763
|
+
child = ", ".join(f"[{c}]" for c in fk.child_columns)
|
|
764
|
+
parent = ", ".join(f"[{c}]" for c in fk.parent_columns)
|
|
765
|
+
out.append(
|
|
766
|
+
f"ALTER TABLE [{schema}].[{fk.child_table}] ADD CONSTRAINT [{fk.name}] FOREIGN KEY "
|
|
767
|
+
f"({child}) REFERENCES [{schema}].[{fk.parent_table}] ({parent})"
|
|
768
|
+
)
|
|
769
|
+
return out
|
|
770
|
+
|
|
771
|
+
|
|
772
|
+
def insert_rows(cursor: Any, table: FakeTable, schema: str = "dbo", batch: int = 500) -> None:
|
|
773
|
+
"""Insert ``table``'s rows through a pyodbc cursor (identity columns keep their values)."""
|
|
774
|
+
if not table.rows:
|
|
775
|
+
return
|
|
776
|
+
cols = ", ".join(f"[{c.name}]" for c in table.columns)
|
|
777
|
+
marks = ", ".join("?" for _ in table.columns)
|
|
778
|
+
# a test helper: schema and table names come from the fixture, never from a user
|
|
779
|
+
sql = f"INSERT INTO [{schema}].[{table.name}] ({cols}) VALUES ({marks})" # nosec B608
|
|
780
|
+
identity = any(c.identity for c in table.columns)
|
|
781
|
+
if identity:
|
|
782
|
+
cursor.execute(f"SET IDENTITY_INSERT [{schema}].[{table.name}] ON")
|
|
783
|
+
for i in range(0, len(table.rows), batch):
|
|
784
|
+
cursor.executemany(sql, table.rows[i : i + batch])
|
|
785
|
+
if identity:
|
|
786
|
+
cursor.execute(f"SET IDENTITY_INSERT [{schema}].[{table.name}] OFF")
|