pgdevkit 0.2.4__py3-none-any.whl → 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pgdevkit/backends/__init__.py +21 -0
- pgdevkit/backends/base.py +27 -0
- pgdevkit/backends/mssql.py +27 -0
- pgdevkit/backends/postgres.py +19 -0
- pgdevkit/cli.py +16 -7
- pgdevkit/db/__init__.py +2 -1
- pgdevkit/db/model.py +9 -2
- pgdevkit/db/mssql_crud.py +290 -0
- pgdevkit/db/mssql_sql.py +48 -0
- pgdevkit/dialect.py +93 -0
- pgdevkit/diff.py +25 -38
- pgdevkit/docs/database-layout.md +15 -0
- pgdevkit/mssql_introspect.py +275 -0
- pgdevkit/parser.py +103 -42
- pgdevkit/testdb/__init__.py +2 -2
- pgdevkit/testdb/_docker.py +41 -0
- pgdevkit/testdb/api.py +36 -4
- pgdevkit/testdb/config.py +13 -0
- pgdevkit/testdb/container.py +1 -34
- pgdevkit/testdb/mssql/__init__.py +1 -0
- pgdevkit/testdb/mssql/api.py +216 -0
- pgdevkit/testdb/mssql/constants.py +60 -0
- pgdevkit/testdb/mssql/container.py +90 -0
- pgdevkit/testdb/query.py +38 -0
- pgdevkit/testdb/schema.py +25 -7
- {pgdevkit-0.2.4.dist-info → pgdevkit-0.3.0.dist-info}/METADATA +57 -5
- pgdevkit-0.3.0.dist-info/RECORD +44 -0
- pgdevkit-0.2.4.dist-info/RECORD +0 -31
- {pgdevkit-0.2.4.dist-info → pgdevkit-0.3.0.dist-info}/WHEEL +0 -0
- {pgdevkit-0.2.4.dist-info → pgdevkit-0.3.0.dist-info}/entry_points.txt +0 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from ..dialect import Dialect, resolve_dialect
|
|
4
|
+
from .base import Backend
|
|
5
|
+
from .mssql import MssqlBackend
|
|
6
|
+
from .postgres import PostgresBackend
|
|
7
|
+
|
|
8
|
+
_REGISTRY: dict[str, Backend] = {
|
|
9
|
+
"postgres": PostgresBackend(),
|
|
10
|
+
"mssql": MssqlBackend(),
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def get_backend(dialect: str | Dialect = "postgres") -> Backend:
|
|
15
|
+
"""Look up the `Backend` for a dialect name (or an already-resolved
|
|
16
|
+
`Dialect`). Defaults to postgres."""
|
|
17
|
+
resolved = resolve_dialect(dialect)
|
|
18
|
+
return _REGISTRY[resolved.name]
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
__all__ = ["Backend", "MssqlBackend", "PostgresBackend", "get_backend"]
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any, Callable, Protocol
|
|
4
|
+
|
|
5
|
+
from ..dialect import Dialect
|
|
6
|
+
from ..models import DatabaseSchema
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class Backend(Protocol):
|
|
10
|
+
"""Introspection + a couple of engine facts, behind one interface.
|
|
11
|
+
|
|
12
|
+
CRUD is deliberately NOT part of this protocol -- psycopg's
|
|
13
|
+
`AsyncConnection` and an MSSQL driver's connection type are unrelated,
|
|
14
|
+
so a unified `backend.retrieve()`/`backend.insert()` surface would force
|
|
15
|
+
existing Postgres callers to go through a new indirection just to keep
|
|
16
|
+
working. Callers that want CRUD import `pgdevkit.db.crud`'s `pg_*`
|
|
17
|
+
functions or `pgdevkit.db.mssql_crud`'s `mssql_*` functions directly,
|
|
18
|
+
exactly as `db/crud.py`'s functions are imported today."""
|
|
19
|
+
|
|
20
|
+
dialect: Dialect
|
|
21
|
+
|
|
22
|
+
def introspect(self, conninfo: str) -> DatabaseSchema: ...
|
|
23
|
+
|
|
24
|
+
def complex_helper_factory(self) -> Callable[..., Any] | None:
|
|
25
|
+
"""A `ComplexHelper`-like factory for composite/enum/JSONB columns,
|
|
26
|
+
or None when the engine has no equivalent (MSSQL)."""
|
|
27
|
+
...
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any, Callable
|
|
4
|
+
|
|
5
|
+
from ..dialect import MSSQL, Dialect
|
|
6
|
+
from ..models import DatabaseSchema
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class MssqlBackend:
|
|
10
|
+
dialect: Dialect = MSSQL
|
|
11
|
+
|
|
12
|
+
def introspect(self, conninfo: str) -> DatabaseSchema:
|
|
13
|
+
# Imported lazily so `import pgdevkit.backends` (and thus
|
|
14
|
+
# `pgdevkit.cli`) doesn't require mssql-python/the mssql extra to
|
|
15
|
+
# be installed unless a caller actually asks for the mssql backend.
|
|
16
|
+
from ..mssql_introspect import introspect_mssql_db
|
|
17
|
+
|
|
18
|
+
return introspect_mssql_db(conninfo)
|
|
19
|
+
|
|
20
|
+
def complex_helper_factory(self) -> Callable[..., Any] | None:
|
|
21
|
+
# MSSQL has no composite type, native enum, or first-class JSONB
|
|
22
|
+
# column type -- there is nothing for a ComplexHelper to adapt.
|
|
23
|
+
# Every `complex_helper` parameter in db/crud.py (and its
|
|
24
|
+
# db/mssql_crud.py counterpart) is already Optional, so callers on
|
|
25
|
+
# this backend simply pass/receive None and every complex-type
|
|
26
|
+
# branch takes its existing no-op path.
|
|
27
|
+
return None
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any, Callable
|
|
4
|
+
|
|
5
|
+
from ..dialect import POSTGRES, Dialect
|
|
6
|
+
from ..introspect import introspect_db
|
|
7
|
+
from ..models import DatabaseSchema
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class PostgresBackend:
|
|
11
|
+
dialect: Dialect = POSTGRES
|
|
12
|
+
|
|
13
|
+
def introspect(self, conninfo: str) -> DatabaseSchema:
|
|
14
|
+
return introspect_db(conninfo)
|
|
15
|
+
|
|
16
|
+
def complex_helper_factory(self) -> Callable[..., Any] | None:
|
|
17
|
+
from ..db.complex_types import ComplexHelper
|
|
18
|
+
|
|
19
|
+
return ComplexHelper
|
pgdevkit/cli.py
CHANGED
|
@@ -10,10 +10,10 @@ from rich.table import Table
|
|
|
10
10
|
from rich import box
|
|
11
11
|
|
|
12
12
|
from . import testdb
|
|
13
|
+
from .backends import get_backend
|
|
13
14
|
from .connection import build_conninfo
|
|
14
15
|
from .diff import DiffKind, compute_diff
|
|
15
16
|
from .fetch_missing import SUBFOLDER, find_missing_objects, layer_folder_for, reconstruct_ddl
|
|
16
|
-
from .introspect import introspect_db
|
|
17
17
|
from .parser import parse_directory
|
|
18
18
|
|
|
19
19
|
app = typer.Typer(name="pgdb", help="PostgreSQL database schema tools")
|
|
@@ -37,9 +37,10 @@ def compare(
|
|
|
37
37
|
None, "--databricks-instance", help="Lakebase instance name (required for Lakebase hosts)"
|
|
38
38
|
),
|
|
39
39
|
report_extra_db: bool = typer.Option(False, "--report-extra-db", help="Report objects in DB but not in scripts"),
|
|
40
|
+
dialect: str = typer.Option("postgres", "--dialect", help="postgres (default) or mssql"),
|
|
40
41
|
scripts_dir: Path = typer.Argument(..., help="Directory containing SQL scripts"),
|
|
41
42
|
) -> None:
|
|
42
|
-
"""Compare SQL scripts to a live
|
|
43
|
+
"""Compare SQL scripts to a live database and report differences."""
|
|
43
44
|
if not scripts_dir.is_dir():
|
|
44
45
|
err_console.print(f"[red]Error:[/red] {scripts_dir} is not a directory")
|
|
45
46
|
raise typer.Exit(2)
|
|
@@ -55,13 +56,19 @@ def compare(
|
|
|
55
56
|
err_console.print(f"[red]Error:[/red] {e}")
|
|
56
57
|
raise typer.Exit(2)
|
|
57
58
|
|
|
59
|
+
try:
|
|
60
|
+
backend = get_backend(dialect)
|
|
61
|
+
except ValueError as e:
|
|
62
|
+
err_console.print(f"[red]Error:[/red] {e}")
|
|
63
|
+
raise typer.Exit(2)
|
|
64
|
+
|
|
58
65
|
with console.status("Parsing SQL scripts..."):
|
|
59
|
-
scripts_schema = parse_directory(scripts_dir)
|
|
66
|
+
scripts_schema = parse_directory(scripts_dir, dialect=backend.dialect)
|
|
60
67
|
|
|
61
68
|
with console.status("Introspecting database..."):
|
|
62
|
-
db_schema =
|
|
69
|
+
db_schema = backend.introspect(conninfo)
|
|
63
70
|
|
|
64
|
-
diffs = compute_diff(scripts_schema, db_schema, report_extra_db=report_extra_db)
|
|
71
|
+
diffs = compute_diff(scripts_schema, db_schema, report_extra_db=report_extra_db, dialect=backend.dialect)
|
|
65
72
|
|
|
66
73
|
if not diffs:
|
|
67
74
|
console.print("[green]No differences found.[/green]")
|
|
@@ -204,8 +211,10 @@ def testdb_status() -> None:
|
|
|
204
211
|
|
|
205
212
|
@testdb_app.command("shell")
|
|
206
213
|
def testdb_shell() -> None:
|
|
207
|
-
"""Drop into psql
|
|
208
|
-
|
|
214
|
+
"""Drop into an interactive shell (psql, or sqlcmd for MSSQL) against
|
|
215
|
+
this workspace's database."""
|
|
216
|
+
binary, argv = testdb.shell_argv()
|
|
217
|
+
os.execvp(binary, argv)
|
|
209
218
|
|
|
210
219
|
|
|
211
220
|
@testdb_app.command("clean")
|
pgdevkit/db/__init__.py
CHANGED
|
@@ -17,13 +17,14 @@ from .crud import (
|
|
|
17
17
|
pg_upsert_many_dict,
|
|
18
18
|
)
|
|
19
19
|
from .loader import SqlLoader
|
|
20
|
-
from .model import PostgresTableModel
|
|
20
|
+
from .model import PostgresTableModel, TableModel
|
|
21
21
|
|
|
22
22
|
__all__ = [
|
|
23
23
|
"ComplexHelper",
|
|
24
24
|
"PgPool",
|
|
25
25
|
"PostgresTableModel",
|
|
26
26
|
"SqlLoader",
|
|
27
|
+
"TableModel",
|
|
27
28
|
"pg_delete",
|
|
28
29
|
"pg_delete_dict",
|
|
29
30
|
"pg_insert",
|
pgdevkit/db/model.py
CHANGED
|
@@ -6,8 +6,10 @@ from typing import Sequence
|
|
|
6
6
|
from pydantic import BaseModel
|
|
7
7
|
|
|
8
8
|
|
|
9
|
-
class
|
|
10
|
-
"""Base class for models that map 1:1 to a database table/row
|
|
9
|
+
class TableModel(BaseModel, ABC):
|
|
10
|
+
"""Base class for models that map 1:1 to a database table/row (any
|
|
11
|
+
engine -- schema/table naming is equally meaningful for Postgres and
|
|
12
|
+
MSSQL, this base class was never actually Postgres-specific).
|
|
11
13
|
|
|
12
14
|
Models representing partial results (joins, aggregations, projections)
|
|
13
15
|
should extend `pydantic.BaseModel` directly instead."""
|
|
@@ -21,3 +23,8 @@ class PostgresTableModel(BaseModel, ABC):
|
|
|
21
23
|
@abstractmethod
|
|
22
24
|
def get_primary_key() -> Sequence[str]:
|
|
23
25
|
"""Return the primary key column name(s)."""
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
# Backward-compat alias -- this class was named PostgresTableModel before
|
|
29
|
+
# MSSQL support existed; kept so existing imports keep working unchanged.
|
|
30
|
+
PostgresTableModel = TableModel
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
from typing import Any, Callable, Mapping, Optional, Sequence, Type, TypeVar
|
|
5
|
+
|
|
6
|
+
from .mssql_sql import ident, json_encode_values, qualified
|
|
7
|
+
from .model import TableModel
|
|
8
|
+
|
|
9
|
+
T = TypeVar("T", bound=TableModel)
|
|
10
|
+
|
|
11
|
+
# `con` below is an mssql-python (github.com/microsoft/mssql-python)
|
|
12
|
+
# connection, but typed as `Any` rather than `mssql_python.Connection` so
|
|
13
|
+
# this module -- and, importantly, the pure `_build_*` query builders
|
|
14
|
+
# below, which have no driver dependency at all -- stays importable (and
|
|
15
|
+
# unit-testable) without the `mssql` extra installed. mssql-python bundles
|
|
16
|
+
# its own ODBC driver, so unlike pyodbc it needs no system driver install;
|
|
17
|
+
# its Connection/Cursor API otherwise mirrors pyodbc's (cursor(), execute(),
|
|
18
|
+
# executemany(), fetchone()/fetchall(), qmark `?` placeholders via a
|
|
19
|
+
# positional params list), which is what `_execute_returning`/`_execute_many`
|
|
20
|
+
# below rely on. `complex_helper` is likewise typed loosely: MSSQL has no
|
|
21
|
+
# ComplexHelper equivalent (see backends/mssql.py), so every caller on this
|
|
22
|
+
# backend passes/receives None here -- the parameter exists purely for
|
|
23
|
+
# signature symmetry with db/crud.py's `pg_*` functions.
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _build_retrieve(table_name: tuple[str, str], pks: dict) -> tuple[str, list]:
|
|
27
|
+
where = " AND ".join(f"{ident(k)} = ?" for k in pks)
|
|
28
|
+
sql = f"SELECT * FROM {qualified(*table_name)} WHERE {where}"
|
|
29
|
+
return sql, list(pks.values())
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _build_retrieve_many(table_name: tuple[str, str], filters: dict) -> tuple[str, list]:
|
|
33
|
+
if not filters:
|
|
34
|
+
return f"SELECT * FROM {qualified(*table_name)}", []
|
|
35
|
+
where = " AND ".join(f"{ident(k)} = ?" for k in filters)
|
|
36
|
+
sql = f"SELECT * FROM {qualified(*table_name)} WHERE {where}"
|
|
37
|
+
return sql, list(filters.values())
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _build_insert(table_name: tuple[str, str], data: dict) -> tuple[str, list]:
|
|
41
|
+
fields = list(data)
|
|
42
|
+
cols = ", ".join(ident(k) for k in fields)
|
|
43
|
+
placeholders = ", ".join("?" for _ in fields)
|
|
44
|
+
sql = f"INSERT INTO {qualified(*table_name)} ({cols}) OUTPUT INSERTED.* VALUES ({placeholders})"
|
|
45
|
+
return sql, [data[k] for k in fields]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _build_insert_many(table_name: tuple[str, str], fields: Sequence[str]) -> str:
|
|
49
|
+
cols = ", ".join(ident(k) for k in fields)
|
|
50
|
+
placeholders = ", ".join("?" for _ in fields)
|
|
51
|
+
return f"INSERT INTO {qualified(*table_name)} ({cols}) VALUES ({placeholders})"
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _build_update(table_name: tuple[str, str], data: dict, primary_keys: Sequence[str]) -> tuple[str, list]:
|
|
55
|
+
set_fields = [k for k in data if k not in primary_keys]
|
|
56
|
+
set_clause = ", ".join(f"{ident(k)} = ?" for k in set_fields)
|
|
57
|
+
where_clause = " AND ".join(f"{ident(pk)} = ?" for pk in primary_keys)
|
|
58
|
+
sql = f"UPDATE {qualified(*table_name)} SET {set_clause} OUTPUT INSERTED.* WHERE {where_clause}"
|
|
59
|
+
params = [data[k] for k in set_fields] + [data[pk] for pk in primary_keys]
|
|
60
|
+
return sql, params
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _build_update_many(table_name: tuple[str, str], fields: Sequence[str], primary_keys: Sequence[str]) -> str:
|
|
64
|
+
set_clause = ", ".join(f"{ident(k)} = ?" for k in fields if k not in primary_keys)
|
|
65
|
+
where_clause = " AND ".join(f"t.{ident(pk)} = ?" for pk in primary_keys)
|
|
66
|
+
return f"UPDATE {qualified(*table_name)} SET {set_clause} WHERE {where_clause}"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _build_upsert_merge(table_name: tuple[str, str], data: dict, primary_keys: Sequence[str]) -> tuple[str, list]:
|
|
70
|
+
"""MERGE INTO ... USING (SELECT ? AS col, ...) AS s ON pk = pk WHEN
|
|
71
|
+
MATCHED THEN UPDATE ... WHEN NOT MATCHED THEN INSERT ... OUTPUT
|
|
72
|
+
INSERTED.* -- the MSSQL replacement for Postgres's `INSERT ... ON
|
|
73
|
+
CONFLICT ... DO UPDATE ... EXCLUDED.col`. Structurally different from
|
|
74
|
+
an upsert-by-string-swap: MERGE is its own statement shape."""
|
|
75
|
+
fields = list(data)
|
|
76
|
+
src_cols = ", ".join(f"? AS {ident(k)}" for k in fields)
|
|
77
|
+
on_clause = " AND ".join(f"t.{ident(pk)} = s.{ident(pk)}" for pk in primary_keys)
|
|
78
|
+
update_fields = [k for k in fields if k not in primary_keys]
|
|
79
|
+
insert_cols = ", ".join(ident(k) for k in fields)
|
|
80
|
+
insert_vals = ", ".join(f"s.{ident(k)}" for k in fields)
|
|
81
|
+
|
|
82
|
+
sql = f"MERGE INTO {qualified(*table_name)} AS t USING (SELECT {src_cols}) AS s ON {on_clause} "
|
|
83
|
+
if update_fields:
|
|
84
|
+
update_clause = ", ".join(f"t.{ident(k)} = s.{ident(k)}" for k in update_fields)
|
|
85
|
+
sql += f"WHEN MATCHED THEN UPDATE SET {update_clause} "
|
|
86
|
+
sql += f"WHEN NOT MATCHED THEN INSERT ({insert_cols}) VALUES ({insert_vals}) OUTPUT INSERTED.*;"
|
|
87
|
+
return sql, [data[k] for k in fields]
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _build_delete(table_name: tuple[str, str], data: dict) -> tuple[str, list]:
|
|
91
|
+
where_clause = " AND ".join(f"{ident(k)} = ?" for k in data)
|
|
92
|
+
sql = f"DELETE FROM {qualified(*table_name)} OUTPUT DELETED.* WHERE {where_clause}"
|
|
93
|
+
return sql, list(data.values())
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
async def _execute_returning(con: Any, sql: str, params: list) -> dict | None:
|
|
97
|
+
def _run() -> dict | None:
|
|
98
|
+
cur = con.cursor()
|
|
99
|
+
try:
|
|
100
|
+
cur.execute(sql, params)
|
|
101
|
+
cols = [c[0] for c in cur.description]
|
|
102
|
+
row = cur.fetchone()
|
|
103
|
+
return dict(zip(cols, row)) if row is not None else None
|
|
104
|
+
finally:
|
|
105
|
+
cur.close()
|
|
106
|
+
|
|
107
|
+
return await asyncio.to_thread(_run)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
async def _execute_many(con: Any, sql: str, param_rows: Sequence[Sequence]) -> None:
|
|
111
|
+
def _run() -> None:
|
|
112
|
+
cur = con.cursor()
|
|
113
|
+
try:
|
|
114
|
+
cur.executemany(sql, list(param_rows))
|
|
115
|
+
finally:
|
|
116
|
+
cur.close()
|
|
117
|
+
|
|
118
|
+
await asyncio.to_thread(_run)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
async def mssql_retrieve(
|
|
122
|
+
con: Any,
|
|
123
|
+
data_type: Type[T],
|
|
124
|
+
pks: dict,
|
|
125
|
+
*,
|
|
126
|
+
complex_helper: Any | None = None,
|
|
127
|
+
) -> T | None:
|
|
128
|
+
"""Fetch a single row by primary key(s). MSSQL has no ComplexHelper
|
|
129
|
+
equivalent (see backends/mssql.py) -- `complex_helper` exists only for
|
|
130
|
+
signature symmetry with `db.crud.pg_retrieve` and is otherwise unused."""
|
|
131
|
+
sql, params = _build_retrieve(data_type.get_table_name(), pks)
|
|
132
|
+
row = await _execute_returning(con, sql, params)
|
|
133
|
+
return data_type(**row) if row else None
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
async def mssql_retrieve_many(
|
|
137
|
+
con: Any,
|
|
138
|
+
data_type: Type[T],
|
|
139
|
+
filters: dict,
|
|
140
|
+
*,
|
|
141
|
+
from_dict: Optional[Callable[[Mapping], T]] = None,
|
|
142
|
+
complex_helper: Any | None = None,
|
|
143
|
+
) -> Sequence[T]:
|
|
144
|
+
"""Fetch multiple rows matching all filter key=value pairs."""
|
|
145
|
+
sql, params = _build_retrieve_many(data_type.get_table_name(), filters)
|
|
146
|
+
|
|
147
|
+
def _run() -> list[dict]:
|
|
148
|
+
cur = con.cursor()
|
|
149
|
+
try:
|
|
150
|
+
cur.execute(sql, params)
|
|
151
|
+
cols = [c[0] for c in cur.description]
|
|
152
|
+
return [dict(zip(cols, row)) for row in cur.fetchall()]
|
|
153
|
+
finally:
|
|
154
|
+
cur.close()
|
|
155
|
+
|
|
156
|
+
rows = await asyncio.to_thread(_run)
|
|
157
|
+
fn = from_dict or (lambda d: data_type(**d))
|
|
158
|
+
return [fn(r) for r in rows]
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
async def mssql_insert(
|
|
162
|
+
con: Any,
|
|
163
|
+
table_name: tuple[str, str],
|
|
164
|
+
data: dict,
|
|
165
|
+
*,
|
|
166
|
+
complex_helper: Any | None = None,
|
|
167
|
+
) -> dict[str, Any]:
|
|
168
|
+
"""Insert one row and return the full row (`OUTPUT INSERTED.*`)."""
|
|
169
|
+
data = json_encode_values(data)
|
|
170
|
+
sql, params = _build_insert(table_name, data)
|
|
171
|
+
row = await _execute_returning(con, sql, params)
|
|
172
|
+
assert row is not None
|
|
173
|
+
return row
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
async def mssql_insert_many(
|
|
177
|
+
con: Any,
|
|
178
|
+
table_name: tuple[str, str],
|
|
179
|
+
data: Sequence[dict],
|
|
180
|
+
*,
|
|
181
|
+
complex_helper: Any | None = None,
|
|
182
|
+
) -> None:
|
|
183
|
+
"""Batch insert -- no OUTPUT, one round-trip via executemany."""
|
|
184
|
+
if not data:
|
|
185
|
+
return
|
|
186
|
+
data = [json_encode_values(row) for row in data]
|
|
187
|
+
fields = list(data[0])
|
|
188
|
+
sql = _build_insert_many(table_name, fields)
|
|
189
|
+
await _execute_many(con, sql, [[row[k] for k in fields] for row in data])
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
async def mssql_update_dict(
|
|
193
|
+
con: Any,
|
|
194
|
+
table_name: tuple[str, str],
|
|
195
|
+
data: dict,
|
|
196
|
+
primary_keys: Sequence[str],
|
|
197
|
+
) -> dict | None:
|
|
198
|
+
"""Update a row identified by primary_keys. Returns the updated row."""
|
|
199
|
+
data = json_encode_values(data)
|
|
200
|
+
sql, params = _build_update(table_name, data, primary_keys)
|
|
201
|
+
return await _execute_returning(con, sql, params)
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
async def mssql_update(con: Any, data: T, data_type: type[T]) -> dict | None:
|
|
205
|
+
"""Update a typed model instance."""
|
|
206
|
+
return await mssql_update_dict(con, data_type.get_table_name(), data.model_dump(), data_type.get_primary_key())
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
async def mssql_upsert_dict(
|
|
210
|
+
con: Any,
|
|
211
|
+
table_name: tuple[str, str],
|
|
212
|
+
data: dict,
|
|
213
|
+
primary_keys: Sequence[str],
|
|
214
|
+
*,
|
|
215
|
+
complex_helper: Any | None = None,
|
|
216
|
+
) -> dict:
|
|
217
|
+
"""MERGE-based upsert, returns the row as a dict."""
|
|
218
|
+
data = json_encode_values(data)
|
|
219
|
+
sql, params = _build_upsert_merge(table_name, data, primary_keys)
|
|
220
|
+
row = await _execute_returning(con, sql, params)
|
|
221
|
+
assert row is not None
|
|
222
|
+
return row
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
async def mssql_upsert(
|
|
226
|
+
con: Any, data: T, data_type: type[T], *, complex_helper: Any | None = None
|
|
227
|
+
) -> dict:
|
|
228
|
+
"""Upsert a typed model instance."""
|
|
229
|
+
return await mssql_upsert_dict(con, data_type.get_table_name(), data.model_dump(), data_type.get_primary_key())
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
async def mssql_upsert_many_dict(
|
|
233
|
+
con: Any,
|
|
234
|
+
table_name: tuple[str, str],
|
|
235
|
+
data: Sequence[dict],
|
|
236
|
+
primary_keys: Sequence[str],
|
|
237
|
+
*,
|
|
238
|
+
must_exist: bool = False,
|
|
239
|
+
complex_helper: Any | None = None,
|
|
240
|
+
) -> None:
|
|
241
|
+
"""Batch upsert.
|
|
242
|
+
|
|
243
|
+
`must_exist=True` switches to a plain UPDATE (no INSERT) matched on
|
|
244
|
+
`primary_keys` -- for callers that only ever update pre-existing rows
|
|
245
|
+
and want a missing row to be a silent no-op rather than create one."""
|
|
246
|
+
if not data:
|
|
247
|
+
return
|
|
248
|
+
data = [json_encode_values(row) for row in data]
|
|
249
|
+
fields = list(data[0])
|
|
250
|
+
if must_exist:
|
|
251
|
+
sql = _build_update_many(table_name, fields, primary_keys)
|
|
252
|
+
non_pk = [k for k in fields if k not in primary_keys]
|
|
253
|
+
rows = [[row[k] for k in non_pk] + [row[pk] for pk in primary_keys] for row in data]
|
|
254
|
+
await _execute_many(con, sql, rows)
|
|
255
|
+
else:
|
|
256
|
+
# MERGE's USING clause is per-row here (first cut) -- a set-based
|
|
257
|
+
# multi-row MERGE ... USING (VALUES (...), (...)) is more efficient
|
|
258
|
+
# but adds real complexity (a dynamic column-count VALUES list);
|
|
259
|
+
# row-by-row via executemany matches how the must_exist branch above
|
|
260
|
+
# already works.
|
|
261
|
+
for row in data:
|
|
262
|
+
sql, params = _build_upsert_merge(table_name, row, primary_keys)
|
|
263
|
+
|
|
264
|
+
def _run() -> None:
|
|
265
|
+
cur = con.cursor()
|
|
266
|
+
try:
|
|
267
|
+
cur.execute(sql, params)
|
|
268
|
+
finally:
|
|
269
|
+
cur.close()
|
|
270
|
+
|
|
271
|
+
await asyncio.to_thread(_run)
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
async def mssql_upsert_many(
|
|
275
|
+
con: Any, data: Sequence[T], data_type: type[T], *, complex_helper: Any | None = None
|
|
276
|
+
) -> None:
|
|
277
|
+
await mssql_upsert_many_dict(con, data_type.get_table_name(), [d.model_dump() for d in data], data_type.get_primary_key())
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
async def mssql_delete_dict(con: Any, table_name: tuple[str, str], data: dict) -> dict | None:
|
|
281
|
+
"""Delete by arbitrary key dict, returns the deleted row."""
|
|
282
|
+
sql, params = _build_delete(table_name, data)
|
|
283
|
+
return await _execute_returning(con, sql, params)
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
async def mssql_delete(con: Any, data: T, data_type: type[T]) -> T | None:
|
|
287
|
+
"""Delete a typed model instance by its primary key(s)."""
|
|
288
|
+
pk_dict = {pk: getattr(data, pk) for pk in data_type.get_primary_key()}
|
|
289
|
+
row = await mssql_delete_dict(con, data_type.get_table_name(), pk_dict)
|
|
290
|
+
return data_type.model_validate(row) if row else None
|
pgdevkit/db/mssql_sql.py
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def ident(name: str) -> str:
|
|
8
|
+
"""Bracket-quote a single identifier, doubling any embedded `]`
|
|
9
|
+
(T-SQL's escaping rule) -- the mssql-python driver has no
|
|
10
|
+
`psycopg.sql.Identifier` equivalent, so this is the composable-SQL
|
|
11
|
+
builder Postgres gets for free, hand-rolled for the one thing it's
|
|
12
|
+
actually needed for here."""
|
|
13
|
+
return f"[{name.replace(']', ']]')}]"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def qualified(schema: str, table: str) -> str:
|
|
17
|
+
return f"{ident(schema)}.{ident(table)}"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def json_encode_value(value: Any) -> Any:
|
|
21
|
+
"""Serialize a value destined for a `json`-typed (or legacy
|
|
22
|
+
`nvarchar(max)`-storing-JSON) column to text.
|
|
23
|
+
|
|
24
|
+
mssql-python has no auto-serialization for dict/list parameter values --
|
|
25
|
+
binding one directly raises `TypeError: Unsupported parameter type`
|
|
26
|
+
(confirmed against mssql-python 1.12.0: its `_map_sql_type` has explicit
|
|
27
|
+
branches for every scalar Python type but none for dict/list, and SQL
|
|
28
|
+
Server's native `json` type -- a genuine first-class type in current
|
|
29
|
+
Azure SQL/SQL Server, unlike the old NVARCHAR(MAX)-plus-OPENJSON()
|
|
30
|
+
convention -- has no dedicated ODBC type code in this driver either, so
|
|
31
|
+
it's fetched back as plain `str`, indistinguishable from any other text
|
|
32
|
+
column). Unlike Postgres, there's no type-registration ambiguity to
|
|
33
|
+
resolve here (composite type vs jsonb vs plain array all need different
|
|
34
|
+
handling there): MSSQL has no composite types, so a Python dict/list
|
|
35
|
+
passed to any MSSQL CRUD call can only sensibly mean "serialize me as
|
|
36
|
+
JSON text" -- no per-column-type lookup needed on the write side.
|
|
37
|
+
|
|
38
|
+
There is deliberately no read-side counterpart: the driver can't tell
|
|
39
|
+
us which columns are `json`-typed (it reports the same opaque `str` for
|
|
40
|
+
those as for a plain `nvarchar`), so auto-parsing fetched values back
|
|
41
|
+
into dict/list would need its own catalog lookup -- a ComplexHelper-like
|
|
42
|
+
mechanism this backend intentionally doesn't have. Callers that know a
|
|
43
|
+
column is JSON deserialize it themselves with `json.loads()`."""
|
|
44
|
+
return json.dumps(value) if isinstance(value, (dict, list)) else value
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def json_encode_values(data: dict) -> dict:
|
|
48
|
+
return {k: json_encode_value(v) for k, v in data.items()}
|
pgdevkit/dialect.py
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
# Postgres type-name synonyms so scripts-vs-db type comparisons in diff.py
|
|
7
|
+
# are spelling-insensitive (e.g. a script written as "int4" matching a
|
|
8
|
+
# catalog-reported "integer"). Moved here (unchanged) from diff.py so both
|
|
9
|
+
# dialects' tables live next to the Dialect they belong to.
|
|
10
|
+
_POSTGRES_TYPE_SYNONYMS = {
|
|
11
|
+
"int": "integer", "int4": "integer",
|
|
12
|
+
"int2": "smallint",
|
|
13
|
+
"int8": "bigint",
|
|
14
|
+
"float4": "real",
|
|
15
|
+
"float8": "double precision",
|
|
16
|
+
"bool": "boolean",
|
|
17
|
+
"decimal": "numeric",
|
|
18
|
+
"varchar": "character varying",
|
|
19
|
+
"char": "character", "bpchar": "character",
|
|
20
|
+
"timestamptz": "timestamp with time zone",
|
|
21
|
+
"timestamp": "timestamp without time zone",
|
|
22
|
+
"timetz": "time with time zone",
|
|
23
|
+
"time": "time without time zone",
|
|
24
|
+
"varbit": "bit varying",
|
|
25
|
+
"serial": "integer", "serial4": "integer",
|
|
26
|
+
"smallserial": "smallint", "serial2": "smallint",
|
|
27
|
+
"bigserial": "bigint", "serial8": "bigint",
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
# T-SQL's ISO/ODBC synonyms (per Microsoft's documented list) plus the one
|
|
31
|
+
# genuinely deprecated pair (timestamp/rowversion) that scripts still use.
|
|
32
|
+
_MSSQL_TYPE_SYNONYMS = {
|
|
33
|
+
"integer": "int",
|
|
34
|
+
"double precision": "float",
|
|
35
|
+
"national character": "nchar",
|
|
36
|
+
"national char": "nchar",
|
|
37
|
+
"national character varying": "nvarchar",
|
|
38
|
+
"national char varying": "nvarchar",
|
|
39
|
+
"char varying": "varchar",
|
|
40
|
+
"binary varying": "varbinary",
|
|
41
|
+
"numeric": "decimal",
|
|
42
|
+
"timestamp": "rowversion",
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass(frozen=True)
|
|
47
|
+
class Dialect:
|
|
48
|
+
"""A thin wrapper around a sqlglot dialect name plus the handful of
|
|
49
|
+
other facts that vary between engines and were previously hardcoded
|
|
50
|
+
throughout parser.py/diff.py/schema.py (default schema, type-name
|
|
51
|
+
synonyms, enum/composite-type support). Intentionally NOT a
|
|
52
|
+
reimplementation of anything sqlglot already does — `sqlglot_name` is
|
|
53
|
+
passed straight through to `sqlglot.parse()`/`.sql(dialect=...)`."""
|
|
54
|
+
|
|
55
|
+
name: str
|
|
56
|
+
sqlglot_name: str
|
|
57
|
+
default_schema: str
|
|
58
|
+
type_synonyms: dict[str, str] = field(default_factory=dict)
|
|
59
|
+
supports_enums: bool = True
|
|
60
|
+
supports_composites: bool = True
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
POSTGRES = Dialect(
|
|
64
|
+
name="postgres",
|
|
65
|
+
sqlglot_name="postgres",
|
|
66
|
+
default_schema="public",
|
|
67
|
+
type_synonyms=_POSTGRES_TYPE_SYNONYMS,
|
|
68
|
+
supports_enums=True,
|
|
69
|
+
supports_composites=True,
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
MSSQL = Dialect(
|
|
73
|
+
name="mssql",
|
|
74
|
+
sqlglot_name="tsql",
|
|
75
|
+
default_schema="dbo",
|
|
76
|
+
type_synonyms=_MSSQL_TYPE_SYNONYMS,
|
|
77
|
+
supports_enums=False,
|
|
78
|
+
supports_composites=False,
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
_REGISTRY = {"postgres": POSTGRES, "mssql": MSSQL}
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def resolve_dialect(dialect: str | Dialect = "postgres") -> Dialect:
|
|
85
|
+
"""Resolve a dialect name (or an already-resolved `Dialect`) to a
|
|
86
|
+
`Dialect` instance. Defaults to postgres, matching every caller's
|
|
87
|
+
default before this module existed."""
|
|
88
|
+
if isinstance(dialect, Dialect):
|
|
89
|
+
return dialect
|
|
90
|
+
try:
|
|
91
|
+
return _REGISTRY[dialect]
|
|
92
|
+
except KeyError:
|
|
93
|
+
raise ValueError(f"Unknown dialect {dialect!r}; expected one of {sorted(_REGISTRY)}") from None
|