sqlalchemy-materialize 0.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sqlalchemy_materialize/__init__.py +0 -0
- sqlalchemy_materialize/base.py +16 -0
- sqlalchemy_materialize/psycopg2.py +253 -0
- sqlalchemy_materialize-0.1.3.dist-info/METADATA +6 -0
- sqlalchemy_materialize-0.1.3.dist-info/RECORD +8 -0
- sqlalchemy_materialize-0.1.3.dist-info/WHEEL +5 -0
- sqlalchemy_materialize-0.1.3.dist-info/entry_points.txt +2 -0
- sqlalchemy_materialize-0.1.3.dist-info/top_level.txt +1 -0
|
File without changes
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
from sqlalchemy.dialects.postgresql.base import PGDialect
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class MaterializeDialect(PGDialect):
|
|
5
|
+
name = "materialize"
|
|
6
|
+
supports_statement_cache = False
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def do_recover_twophase(self, connection):
|
|
10
|
+
'''
|
|
11
|
+
Materialize does not support two-phase commit recovery
|
|
12
|
+
'''
|
|
13
|
+
|
|
14
|
+
raise NotImplementedError(
|
|
15
|
+
"Materialize does not support two-phase commit recovery"
|
|
16
|
+
)
|
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
from sqlalchemy import bindparam, text
|
|
2
|
+
|
|
3
|
+
from sqlalchemy.dialects.postgresql.psycopg2 import PGDialect_psycopg2
|
|
4
|
+
from sqlalchemy.engine.reflection import ObjectKind, ObjectScope
|
|
5
|
+
|
|
6
|
+
from .base import MaterializeDialect
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class MaterializeDialect_psycopg2(
|
|
10
|
+
MaterializeDialect,
|
|
11
|
+
PGDialect_psycopg2,
|
|
12
|
+
):
|
|
13
|
+
driver = "psycopg2"
|
|
14
|
+
supports_statement_cache = False
|
|
15
|
+
|
|
16
|
+
# Inherited single-relation reflection methods delegate to get_multi_* methods.
|
|
17
|
+
|
|
18
|
+
def _get_relation_ids(self, connection, schema, filter_names, scope, kind):
|
|
19
|
+
"""Return a name-to-ID mapping filtered by schema, names, scope, and kind.
|
|
20
|
+
|
|
21
|
+
Prefer temporary relations when matching names overlap.
|
|
22
|
+
"""
|
|
23
|
+
catalogs = [
|
|
24
|
+
catalog
|
|
25
|
+
for object_kind, catalog in (
|
|
26
|
+
(ObjectKind.TABLE, "mz_tables"),
|
|
27
|
+
(ObjectKind.VIEW, "mz_views"),
|
|
28
|
+
(ObjectKind.MATERIALIZED_VIEW, "mz_materialized_views"),
|
|
29
|
+
)
|
|
30
|
+
if object_kind in kind
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
if not catalogs or not scope:
|
|
34
|
+
return {}
|
|
35
|
+
|
|
36
|
+
# Temporary relations use schema_id '0' and need an explicit match
|
|
37
|
+
# because pg_table_is_visible returns NULL for them.
|
|
38
|
+
|
|
39
|
+
schema_filter = (
|
|
40
|
+
"(r.schema_id = '0' OR pg_table_is_visible(r.oid))"
|
|
41
|
+
if schema is None
|
|
42
|
+
else "r.schema_id = '0'"
|
|
43
|
+
if schema == "mz_temp"
|
|
44
|
+
else "s.name = :schema"
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
scope_filter = "TRUE"
|
|
48
|
+
if scope == ObjectScope.DEFAULT:
|
|
49
|
+
scope_filter = "r.schema_id <> '0'"
|
|
50
|
+
elif scope == ObjectScope.TEMPORARY:
|
|
51
|
+
scope_filter = "r.schema_id = '0'"
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
# Catalog identifiers come only from the fixed mapping above.
|
|
55
|
+
relations_sql = " UNION ALL ".join(
|
|
56
|
+
f"SELECT id, oid, name, schema_id FROM mz_catalog.{catalog}"
|
|
57
|
+
for catalog in catalogs
|
|
58
|
+
)
|
|
59
|
+
existing_relations = connection.execute(text(f"""
|
|
60
|
+
SELECT r.id, r.name, r.schema_id = '0' AS is_temp
|
|
61
|
+
FROM ({relations_sql}) AS r
|
|
62
|
+
LEFT JOIN mz_catalog.mz_schemas AS s ON r.schema_id = s.id
|
|
63
|
+
LEFT JOIN mz_catalog.mz_databases AS d ON s.database_id = d.id
|
|
64
|
+
WHERE ({schema_filter}) AND ({scope_filter})
|
|
65
|
+
AND (d.name = current_database()
|
|
66
|
+
OR r.schema_id = '0')
|
|
67
|
+
"""), {"schema": schema}).mappings().all()
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
# Temporary relations take precedence when names overlap.
|
|
71
|
+
|
|
72
|
+
relation_id_by_name = {}
|
|
73
|
+
for row in existing_relations:
|
|
74
|
+
if row["name"] in relation_id_by_name:
|
|
75
|
+
relation_id_by_name[row["name"]] = row["id"] if row["is_temp"] else relation_id_by_name[row["name"]]
|
|
76
|
+
else:
|
|
77
|
+
relation_id_by_name[row["name"]] = row["id"]
|
|
78
|
+
|
|
79
|
+
if filter_names is None:
|
|
80
|
+
filter_names = sorted(relation_id_by_name)
|
|
81
|
+
else:
|
|
82
|
+
filter_names = sorted(set(relation_id_by_name).intersection(filter_names))
|
|
83
|
+
|
|
84
|
+
return {name: relation_id_by_name[name] for name in filter_names}
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def get_multi_columns(self, connection, schema, filter_names, scope, kind, **kw):
|
|
88
|
+
"""Return ((schema, relation_name), columns) pairs for matching relations."""
|
|
89
|
+
relation_id_by_name = self._get_relation_ids(
|
|
90
|
+
connection, schema, filter_names, scope, kind
|
|
91
|
+
)
|
|
92
|
+
if not relation_id_by_name:
|
|
93
|
+
return {}.items()
|
|
94
|
+
relation_ids = list(relation_id_by_name.values())
|
|
95
|
+
name_by_id = {rel_id: name for name, rel_id in relation_id_by_name.items()}
|
|
96
|
+
columns = {(schema, name): [] for name in relation_id_by_name}
|
|
97
|
+
|
|
98
|
+
query = text("""
|
|
99
|
+
SELECT
|
|
100
|
+
id, name, format_type(type_oid, type_mod) AS type, nullable, "default"
|
|
101
|
+
FROM mz_catalog.mz_columns
|
|
102
|
+
WHERE id IN :relation_ids
|
|
103
|
+
ORDER BY id, position
|
|
104
|
+
""").bindparams(bindparam("relation_ids", expanding=True))
|
|
105
|
+
|
|
106
|
+
result = connection.execute(query, {
|
|
107
|
+
"relation_ids": relation_ids,
|
|
108
|
+
}).mappings().all()
|
|
109
|
+
|
|
110
|
+
for row in result:
|
|
111
|
+
relation_name = name_by_id[row["id"]]
|
|
112
|
+
columns[(schema, relation_name)].append({
|
|
113
|
+
"name": row["name"],
|
|
114
|
+
# This parser converts type strings without querying PG catalogs.
|
|
115
|
+
# Unknown Materialize-specific types warn and become NullType.
|
|
116
|
+
"type": self._reflect_type(
|
|
117
|
+
row["type"], {}, {},
|
|
118
|
+
type_description=f"column '{relation_name}.{row['name']}'",
|
|
119
|
+
collation=None,
|
|
120
|
+
),
|
|
121
|
+
"nullable": bool(row["nullable"]),
|
|
122
|
+
"default": row["default"],
|
|
123
|
+
})
|
|
124
|
+
return columns.items()
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def get_multi_indexes(
|
|
128
|
+
self, connection, schema, filter_names, scope, kind, **kw
|
|
129
|
+
):
|
|
130
|
+
"""Reflect indexes for relations matching the requested filters.
|
|
131
|
+
|
|
132
|
+
@param connection: SQLAlchemy connection used to query the catalogs.
|
|
133
|
+
@param schema: Schema to inspect, or None for unqualified-name lookup.
|
|
134
|
+
@param filter_names: Relation names to include, or None for all relations
|
|
135
|
+
matching the schema, scope, and kind filters.
|
|
136
|
+
@param scope: ObjectScope flag selecting non-temporary objects,
|
|
137
|
+
temporary objects, or both.
|
|
138
|
+
@param kind: ObjectKind flag selecting tables, views, materialized views,
|
|
139
|
+
or a combination of these kinds.
|
|
140
|
+
@param kw: Additional keyword arguments supplied by SQLAlchemy.
|
|
141
|
+
@return: An iterable of ((schema, table_name), indexes) pairs, where
|
|
142
|
+
indexes is a list of reflected index dictionaries. Relations without
|
|
143
|
+
indexes are included with an empty list.
|
|
144
|
+
"""
|
|
145
|
+
|
|
146
|
+
relation_id_by_name = self._get_relation_ids(
|
|
147
|
+
connection, schema, filter_names, scope, kind
|
|
148
|
+
)
|
|
149
|
+
if not relation_id_by_name:
|
|
150
|
+
return {}.items()
|
|
151
|
+
filter_names = list(relation_id_by_name)
|
|
152
|
+
relation_ids = list(relation_id_by_name.values())
|
|
153
|
+
|
|
154
|
+
query = text("""
|
|
155
|
+
SELECT
|
|
156
|
+
r.name AS table_name,
|
|
157
|
+
i.id AS index_id,
|
|
158
|
+
i.name AS index_name,
|
|
159
|
+
ic.index_position,
|
|
160
|
+
col.name AS column_name,
|
|
161
|
+
ic.on_expression
|
|
162
|
+
FROM mz_catalog.mz_indexes AS i
|
|
163
|
+
JOIN mz_catalog.mz_relations AS r
|
|
164
|
+
ON i.on_id = r.id
|
|
165
|
+
JOIN mz_catalog.mz_index_columns AS ic
|
|
166
|
+
ON ic.index_id = i.id
|
|
167
|
+
LEFT JOIN mz_catalog.mz_columns AS col
|
|
168
|
+
ON col.id = r.id
|
|
169
|
+
AND col.position = ic.on_position
|
|
170
|
+
WHERE r.id IN :relation_ids
|
|
171
|
+
ORDER BY r.name, i.id, ic.index_position
|
|
172
|
+
""").bindparams(bindparam("relation_ids", expanding=True))
|
|
173
|
+
|
|
174
|
+
result = connection.execute(query, {
|
|
175
|
+
"relation_ids": relation_ids,
|
|
176
|
+
}).mappings().all()
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
parsed = {}
|
|
180
|
+
prev_id = None
|
|
181
|
+
prev_table_name = None
|
|
182
|
+
|
|
183
|
+
for row in result:
|
|
184
|
+
table_name = row["table_name"]
|
|
185
|
+
index_id = row["index_id"]
|
|
186
|
+
name = row["index_name"]
|
|
187
|
+
col_name = row["column_name"]
|
|
188
|
+
on_exp = row["on_expression"]
|
|
189
|
+
|
|
190
|
+
# Mixed indexes need an expression entry for each indexed column.
|
|
191
|
+
on_exp = on_exp if on_exp is not None else col_name
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
if prev_table_name != table_name:
|
|
195
|
+
parsed[(schema, table_name)] = []
|
|
196
|
+
|
|
197
|
+
# The query orders rows by table, index, and index column position.
|
|
198
|
+
if prev_id != index_id:
|
|
199
|
+
parsed[(schema, table_name)].append(
|
|
200
|
+
{
|
|
201
|
+
"name": name,
|
|
202
|
+
"column_names": [],
|
|
203
|
+
"expressions": [],
|
|
204
|
+
"unique": False,
|
|
205
|
+
}
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
parsed[(schema, table_name)][-1]["column_names"].append(col_name)
|
|
210
|
+
parsed[(schema, table_name)][-1]["expressions"].append(on_exp)
|
|
211
|
+
|
|
212
|
+
prev_id = index_id
|
|
213
|
+
prev_table_name = table_name
|
|
214
|
+
|
|
215
|
+
# Only expression indexes need expressions; plain columns suffice otherwise.
|
|
216
|
+
for indexes in parsed.values():
|
|
217
|
+
for index in indexes:
|
|
218
|
+
if None not in index["column_names"]:
|
|
219
|
+
del index["expressions"]
|
|
220
|
+
|
|
221
|
+
# Include matching relations even when the index query returned no rows.
|
|
222
|
+
for table_name in filter_names:
|
|
223
|
+
if (schema, table_name) not in parsed:
|
|
224
|
+
parsed[(schema, table_name)] = []
|
|
225
|
+
|
|
226
|
+
return parsed.items()
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def get_multi_check_constraints(self, connection, schema, filter_names, scope, kind, **kw):
|
|
230
|
+
'''
|
|
231
|
+
Materialize doesn't support constraint for tables built in Materialize.
|
|
232
|
+
For other database like PostgreSQL, MySQL that is upstreamed to Materialize,
|
|
233
|
+
you should connect directly to it with its corresponding SQLALchemy dialect
|
|
234
|
+
'''
|
|
235
|
+
relation_id_by_name = self._get_relation_ids(
|
|
236
|
+
connection, schema, filter_names, scope, kind
|
|
237
|
+
)
|
|
238
|
+
return {(schema, name): [] for name in relation_id_by_name}.items()
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def get_multi_foreign_keys(
|
|
242
|
+
self, connection, schema, filter_names, scope, kind,
|
|
243
|
+
postgresql_ignore_search_path=False, **kw
|
|
244
|
+
):
|
|
245
|
+
'''
|
|
246
|
+
Materialize doesn't support foreign key for tables built in Materialize.
|
|
247
|
+
For other database like PostgreSQL, MySQL that is upstreamed to Materialize,
|
|
248
|
+
you should connect directly to it with its corresponding SQLALchemy dialect
|
|
249
|
+
'''
|
|
250
|
+
relation_id_by_name = self._get_relation_ids(
|
|
251
|
+
connection, schema, filter_names, scope, kind
|
|
252
|
+
)
|
|
253
|
+
return {(schema, name): [] for name in relation_id_by_name}.items()
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
sqlalchemy_materialize/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
sqlalchemy_materialize/base.py,sha256=Uy-vftSKNcIGx7Uv9qT-QyReKY_i-gpwFGX9EAA46p8,418
|
|
3
|
+
sqlalchemy_materialize/psycopg2.py,sha256=VyThw4u_gCx43TXLxoUZxZ8SuudUrVbleXRKd5v9o2M,10084
|
|
4
|
+
sqlalchemy_materialize-0.1.3.dist-info/METADATA,sha256=FvBUKNremW0Y8wb1DvwIhqzZM1w5R5fo3uFuQNLYs9E,154
|
|
5
|
+
sqlalchemy_materialize-0.1.3.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
6
|
+
sqlalchemy_materialize-0.1.3.dist-info/entry_points.txt,sha256=QGtOOGpU3ut9AcyRpvyXQNXgBhIwv03USKbtoDPLR6U,105
|
|
7
|
+
sqlalchemy_materialize-0.1.3.dist-info/top_level.txt,sha256=MvyyFKQaC-AXkKLIhCuw6MRrgqbtF72XLkW7DexW-5k,23
|
|
8
|
+
sqlalchemy_materialize-0.1.3.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
sqlalchemy_materialize
|