sqlalchemy-materialize 0.1.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sqlalchemy_materialize-0.1.3/PKG-INFO +6 -0
- sqlalchemy_materialize-0.1.3/README.md +0 -0
- sqlalchemy_materialize-0.1.3/pyproject.toml +3 -0
- sqlalchemy_materialize-0.1.3/setup.cfg +4 -0
- sqlalchemy_materialize-0.1.3/setup.py +16 -0
- sqlalchemy_materialize-0.1.3/sqlalchemy_materialize/__init__.py +0 -0
- sqlalchemy_materialize-0.1.3/sqlalchemy_materialize/base.py +16 -0
- sqlalchemy_materialize-0.1.3/sqlalchemy_materialize/psycopg2.py +253 -0
- sqlalchemy_materialize-0.1.3/sqlalchemy_materialize.egg-info/PKG-INFO +6 -0
- sqlalchemy_materialize-0.1.3/sqlalchemy_materialize.egg-info/SOURCES.txt +15 -0
- sqlalchemy_materialize-0.1.3/sqlalchemy_materialize.egg-info/dependency_links.txt +1 -0
- sqlalchemy_materialize-0.1.3/sqlalchemy_materialize.egg-info/entry_points.txt +2 -0
- sqlalchemy_materialize-0.1.3/sqlalchemy_materialize.egg-info/requires.txt +2 -0
- sqlalchemy_materialize-0.1.3/sqlalchemy_materialize.egg-info/top_level.txt +1 -0
- sqlalchemy_materialize-0.1.3/tests/test_column_reflection.py +67 -0
- sqlalchemy_materialize-0.1.3/tests/test_dialect.py +14 -0
- sqlalchemy_materialize-0.1.3/tests/test_index_scope.py +167 -0
|
File without changes
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
from setuptools import find_packages, setup
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
# Increment the version every upload
|
|
5
|
+
setup(
|
|
6
|
+
name="sqlalchemy-materialize",
|
|
7
|
+
version="0.1.3",
|
|
8
|
+
packages=find_packages(),
|
|
9
|
+
install_requires=["SQLAlchemy>=2.0,<2.1", "psycopg2>=2.9"],
|
|
10
|
+
entry_points={
|
|
11
|
+
"sqlalchemy.dialects": [
|
|
12
|
+
"materialize.psycopg2 = "
|
|
13
|
+
"sqlalchemy_materialize.psycopg2:MaterializeDialect_psycopg2",
|
|
14
|
+
],
|
|
15
|
+
},
|
|
16
|
+
)
|
|
File without changes
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
from sqlalchemy.dialects.postgresql.base import PGDialect
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class MaterializeDialect(PGDialect):
|
|
5
|
+
name = "materialize"
|
|
6
|
+
supports_statement_cache = False
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def do_recover_twophase(self, connection):
|
|
10
|
+
'''
|
|
11
|
+
Materialize does not support two-phase commit recovery
|
|
12
|
+
'''
|
|
13
|
+
|
|
14
|
+
raise NotImplementedError(
|
|
15
|
+
"Materialize does not support two-phase commit recovery"
|
|
16
|
+
)
|
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
from sqlalchemy import bindparam, text
|
|
2
|
+
|
|
3
|
+
from sqlalchemy.dialects.postgresql.psycopg2 import PGDialect_psycopg2
|
|
4
|
+
from sqlalchemy.engine.reflection import ObjectKind, ObjectScope
|
|
5
|
+
|
|
6
|
+
from .base import MaterializeDialect
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class MaterializeDialect_psycopg2(
|
|
10
|
+
MaterializeDialect,
|
|
11
|
+
PGDialect_psycopg2,
|
|
12
|
+
):
|
|
13
|
+
driver = "psycopg2"
|
|
14
|
+
supports_statement_cache = False
|
|
15
|
+
|
|
16
|
+
# Inherited single-relation reflection methods delegate to get_multi_* methods.
|
|
17
|
+
|
|
18
|
+
def _get_relation_ids(self, connection, schema, filter_names, scope, kind):
|
|
19
|
+
"""Return a name-to-ID mapping filtered by schema, names, scope, and kind.
|
|
20
|
+
|
|
21
|
+
Prefer temporary relations when matching names overlap.
|
|
22
|
+
"""
|
|
23
|
+
catalogs = [
|
|
24
|
+
catalog
|
|
25
|
+
for object_kind, catalog in (
|
|
26
|
+
(ObjectKind.TABLE, "mz_tables"),
|
|
27
|
+
(ObjectKind.VIEW, "mz_views"),
|
|
28
|
+
(ObjectKind.MATERIALIZED_VIEW, "mz_materialized_views"),
|
|
29
|
+
)
|
|
30
|
+
if object_kind in kind
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
if not catalogs or not scope:
|
|
34
|
+
return {}
|
|
35
|
+
|
|
36
|
+
# Temporary relations use schema_id '0' and need an explicit match
|
|
37
|
+
# because pg_table_is_visible returns NULL for them.
|
|
38
|
+
|
|
39
|
+
schema_filter = (
|
|
40
|
+
"(r.schema_id = '0' OR pg_table_is_visible(r.oid))"
|
|
41
|
+
if schema is None
|
|
42
|
+
else "r.schema_id = '0'"
|
|
43
|
+
if schema == "mz_temp"
|
|
44
|
+
else "s.name = :schema"
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
scope_filter = "TRUE"
|
|
48
|
+
if scope == ObjectScope.DEFAULT:
|
|
49
|
+
scope_filter = "r.schema_id <> '0'"
|
|
50
|
+
elif scope == ObjectScope.TEMPORARY:
|
|
51
|
+
scope_filter = "r.schema_id = '0'"
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
# Catalog identifiers come only from the fixed mapping above.
|
|
55
|
+
relations_sql = " UNION ALL ".join(
|
|
56
|
+
f"SELECT id, oid, name, schema_id FROM mz_catalog.{catalog}"
|
|
57
|
+
for catalog in catalogs
|
|
58
|
+
)
|
|
59
|
+
existing_relations = connection.execute(text(f"""
|
|
60
|
+
SELECT r.id, r.name, r.schema_id = '0' AS is_temp
|
|
61
|
+
FROM ({relations_sql}) AS r
|
|
62
|
+
LEFT JOIN mz_catalog.mz_schemas AS s ON r.schema_id = s.id
|
|
63
|
+
LEFT JOIN mz_catalog.mz_databases AS d ON s.database_id = d.id
|
|
64
|
+
WHERE ({schema_filter}) AND ({scope_filter})
|
|
65
|
+
AND (d.name = current_database()
|
|
66
|
+
OR r.schema_id = '0')
|
|
67
|
+
"""), {"schema": schema}).mappings().all()
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
# Temporary relations take precedence when names overlap.
|
|
71
|
+
|
|
72
|
+
relation_id_by_name = {}
|
|
73
|
+
for row in existing_relations:
|
|
74
|
+
if row["name"] in relation_id_by_name:
|
|
75
|
+
relation_id_by_name[row["name"]] = row["id"] if row["is_temp"] else relation_id_by_name[row["name"]]
|
|
76
|
+
else:
|
|
77
|
+
relation_id_by_name[row["name"]] = row["id"]
|
|
78
|
+
|
|
79
|
+
if filter_names is None:
|
|
80
|
+
filter_names = sorted(relation_id_by_name)
|
|
81
|
+
else:
|
|
82
|
+
filter_names = sorted(set(relation_id_by_name).intersection(filter_names))
|
|
83
|
+
|
|
84
|
+
return {name: relation_id_by_name[name] for name in filter_names}
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def get_multi_columns(self, connection, schema, filter_names, scope, kind, **kw):
|
|
88
|
+
"""Return ((schema, relation_name), columns) pairs for matching relations."""
|
|
89
|
+
relation_id_by_name = self._get_relation_ids(
|
|
90
|
+
connection, schema, filter_names, scope, kind
|
|
91
|
+
)
|
|
92
|
+
if not relation_id_by_name:
|
|
93
|
+
return {}.items()
|
|
94
|
+
relation_ids = list(relation_id_by_name.values())
|
|
95
|
+
name_by_id = {rel_id: name for name, rel_id in relation_id_by_name.items()}
|
|
96
|
+
columns = {(schema, name): [] for name in relation_id_by_name}
|
|
97
|
+
|
|
98
|
+
query = text("""
|
|
99
|
+
SELECT
|
|
100
|
+
id, name, format_type(type_oid, type_mod) AS type, nullable, "default"
|
|
101
|
+
FROM mz_catalog.mz_columns
|
|
102
|
+
WHERE id IN :relation_ids
|
|
103
|
+
ORDER BY id, position
|
|
104
|
+
""").bindparams(bindparam("relation_ids", expanding=True))
|
|
105
|
+
|
|
106
|
+
result = connection.execute(query, {
|
|
107
|
+
"relation_ids": relation_ids,
|
|
108
|
+
}).mappings().all()
|
|
109
|
+
|
|
110
|
+
for row in result:
|
|
111
|
+
relation_name = name_by_id[row["id"]]
|
|
112
|
+
columns[(schema, relation_name)].append({
|
|
113
|
+
"name": row["name"],
|
|
114
|
+
# This parser converts type strings without querying PG catalogs.
|
|
115
|
+
# Unknown Materialize-specific types warn and become NullType.
|
|
116
|
+
"type": self._reflect_type(
|
|
117
|
+
row["type"], {}, {},
|
|
118
|
+
type_description=f"column '{relation_name}.{row['name']}'",
|
|
119
|
+
collation=None,
|
|
120
|
+
),
|
|
121
|
+
"nullable": bool(row["nullable"]),
|
|
122
|
+
"default": row["default"],
|
|
123
|
+
})
|
|
124
|
+
return columns.items()
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def get_multi_indexes(
|
|
128
|
+
self, connection, schema, filter_names, scope, kind, **kw
|
|
129
|
+
):
|
|
130
|
+
"""Reflect indexes for relations matching the requested filters.
|
|
131
|
+
|
|
132
|
+
@param connection: SQLAlchemy connection used to query the catalogs.
|
|
133
|
+
@param schema: Schema to inspect, or None for unqualified-name lookup.
|
|
134
|
+
@param filter_names: Relation names to include, or None for all relations
|
|
135
|
+
matching the schema, scope, and kind filters.
|
|
136
|
+
@param scope: ObjectScope flag selecting non-temporary objects,
|
|
137
|
+
temporary objects, or both.
|
|
138
|
+
@param kind: ObjectKind flag selecting tables, views, materialized views,
|
|
139
|
+
or a combination of these kinds.
|
|
140
|
+
@param kw: Additional keyword arguments supplied by SQLAlchemy.
|
|
141
|
+
@return: An iterable of ((schema, table_name), indexes) pairs, where
|
|
142
|
+
indexes is a list of reflected index dictionaries. Relations without
|
|
143
|
+
indexes are included with an empty list.
|
|
144
|
+
"""
|
|
145
|
+
|
|
146
|
+
relation_id_by_name = self._get_relation_ids(
|
|
147
|
+
connection, schema, filter_names, scope, kind
|
|
148
|
+
)
|
|
149
|
+
if not relation_id_by_name:
|
|
150
|
+
return {}.items()
|
|
151
|
+
filter_names = list(relation_id_by_name)
|
|
152
|
+
relation_ids = list(relation_id_by_name.values())
|
|
153
|
+
|
|
154
|
+
query = text("""
|
|
155
|
+
SELECT
|
|
156
|
+
r.name AS table_name,
|
|
157
|
+
i.id AS index_id,
|
|
158
|
+
i.name AS index_name,
|
|
159
|
+
ic.index_position,
|
|
160
|
+
col.name AS column_name,
|
|
161
|
+
ic.on_expression
|
|
162
|
+
FROM mz_catalog.mz_indexes AS i
|
|
163
|
+
JOIN mz_catalog.mz_relations AS r
|
|
164
|
+
ON i.on_id = r.id
|
|
165
|
+
JOIN mz_catalog.mz_index_columns AS ic
|
|
166
|
+
ON ic.index_id = i.id
|
|
167
|
+
LEFT JOIN mz_catalog.mz_columns AS col
|
|
168
|
+
ON col.id = r.id
|
|
169
|
+
AND col.position = ic.on_position
|
|
170
|
+
WHERE r.id IN :relation_ids
|
|
171
|
+
ORDER BY r.name, i.id, ic.index_position
|
|
172
|
+
""").bindparams(bindparam("relation_ids", expanding=True))
|
|
173
|
+
|
|
174
|
+
result = connection.execute(query, {
|
|
175
|
+
"relation_ids": relation_ids,
|
|
176
|
+
}).mappings().all()
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
parsed = {}
|
|
180
|
+
prev_id = None
|
|
181
|
+
prev_table_name = None
|
|
182
|
+
|
|
183
|
+
for row in result:
|
|
184
|
+
table_name = row["table_name"]
|
|
185
|
+
index_id = row["index_id"]
|
|
186
|
+
name = row["index_name"]
|
|
187
|
+
col_name = row["column_name"]
|
|
188
|
+
on_exp = row["on_expression"]
|
|
189
|
+
|
|
190
|
+
# Mixed indexes need an expression entry for each indexed column.
|
|
191
|
+
on_exp = on_exp if on_exp is not None else col_name
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
if prev_table_name != table_name:
|
|
195
|
+
parsed[(schema, table_name)] = []
|
|
196
|
+
|
|
197
|
+
# The query orders rows by table, index, and index column position.
|
|
198
|
+
if prev_id != index_id:
|
|
199
|
+
parsed[(schema, table_name)].append(
|
|
200
|
+
{
|
|
201
|
+
"name": name,
|
|
202
|
+
"column_names": [],
|
|
203
|
+
"expressions": [],
|
|
204
|
+
"unique": False,
|
|
205
|
+
}
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
parsed[(schema, table_name)][-1]["column_names"].append(col_name)
|
|
210
|
+
parsed[(schema, table_name)][-1]["expressions"].append(on_exp)
|
|
211
|
+
|
|
212
|
+
prev_id = index_id
|
|
213
|
+
prev_table_name = table_name
|
|
214
|
+
|
|
215
|
+
# Only expression indexes need expressions; plain columns suffice otherwise.
|
|
216
|
+
for indexes in parsed.values():
|
|
217
|
+
for index in indexes:
|
|
218
|
+
if None not in index["column_names"]:
|
|
219
|
+
del index["expressions"]
|
|
220
|
+
|
|
221
|
+
# Include matching relations even when the index query returned no rows.
|
|
222
|
+
for table_name in filter_names:
|
|
223
|
+
if (schema, table_name) not in parsed:
|
|
224
|
+
parsed[(schema, table_name)] = []
|
|
225
|
+
|
|
226
|
+
return parsed.items()
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def get_multi_check_constraints(self, connection, schema, filter_names, scope, kind, **kw):
|
|
230
|
+
'''
|
|
231
|
+
Materialize doesn't support constraint for tables built in Materialize.
|
|
232
|
+
For other database like PostgreSQL, MySQL that is upstreamed to Materialize,
|
|
233
|
+
you should connect directly to it with its corresponding SQLALchemy dialect
|
|
234
|
+
'''
|
|
235
|
+
relation_id_by_name = self._get_relation_ids(
|
|
236
|
+
connection, schema, filter_names, scope, kind
|
|
237
|
+
)
|
|
238
|
+
return {(schema, name): [] for name in relation_id_by_name}.items()
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def get_multi_foreign_keys(
|
|
242
|
+
self, connection, schema, filter_names, scope, kind,
|
|
243
|
+
postgresql_ignore_search_path=False, **kw
|
|
244
|
+
):
|
|
245
|
+
'''
|
|
246
|
+
Materialize doesn't support foreign key for tables built in Materialize.
|
|
247
|
+
For other database like PostgreSQL, MySQL that is upstreamed to Materialize,
|
|
248
|
+
you should connect directly to it with its corresponding SQLALchemy dialect
|
|
249
|
+
'''
|
|
250
|
+
relation_id_by_name = self._get_relation_ids(
|
|
251
|
+
connection, schema, filter_names, scope, kind
|
|
252
|
+
)
|
|
253
|
+
return {(schema, name): [] for name in relation_id_by_name}.items()
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
setup.py
|
|
4
|
+
sqlalchemy_materialize/__init__.py
|
|
5
|
+
sqlalchemy_materialize/base.py
|
|
6
|
+
sqlalchemy_materialize/psycopg2.py
|
|
7
|
+
sqlalchemy_materialize.egg-info/PKG-INFO
|
|
8
|
+
sqlalchemy_materialize.egg-info/SOURCES.txt
|
|
9
|
+
sqlalchemy_materialize.egg-info/dependency_links.txt
|
|
10
|
+
sqlalchemy_materialize.egg-info/entry_points.txt
|
|
11
|
+
sqlalchemy_materialize.egg-info/requires.txt
|
|
12
|
+
sqlalchemy_materialize.egg-info/top_level.txt
|
|
13
|
+
tests/test_column_reflection.py
|
|
14
|
+
tests/test_dialect.py
|
|
15
|
+
tests/test_index_scope.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
sqlalchemy_materialize
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
from unittest.mock import Mock
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
from sqlalchemy import Integer, Numeric, String, create_engine
|
|
5
|
+
from sqlalchemy.dialects.postgresql import ARRAY
|
|
6
|
+
from sqlalchemy.engine.reflection import ObjectKind, ObjectScope
|
|
7
|
+
from sqlalchemy.exc import SAWarning
|
|
8
|
+
from sqlalchemy.sql.sqltypes import NullType
|
|
9
|
+
|
|
10
|
+
from sqlalchemy_materialize.psycopg2 import MaterializeDialect_psycopg2
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def test_columns_by_relation_id():
|
|
14
|
+
engine = create_engine("sqlite://")
|
|
15
|
+
dialect = MaterializeDialect_psycopg2()
|
|
16
|
+
dialect._get_relation_ids = Mock(return_value={"users": "u1", "view": "u2", "empty": "u3"})
|
|
17
|
+
with engine.connect() as connection:
|
|
18
|
+
# Stand in for the server formatter; verify both OID and modifier are
|
|
19
|
+
# supplied and that the formatted result reaches SQLAlchemy's parser.
|
|
20
|
+
formatted_types = {
|
|
21
|
+
(1700, 655366): "numeric(10,2)",
|
|
22
|
+
(23, -1): "integer",
|
|
23
|
+
(1015, 24): "character varying(20)[]",
|
|
24
|
+
(99999, -1): "custom_type",
|
|
25
|
+
}
|
|
26
|
+
connection.connection.driver_connection.create_function(
|
|
27
|
+
"format_type", 2, lambda oid, modifier: formatted_types[(oid, modifier)]
|
|
28
|
+
)
|
|
29
|
+
connection.exec_driver_sql("ATTACH DATABASE ':memory:' AS mz_catalog")
|
|
30
|
+
connection.exec_driver_sql('CREATE TABLE mz_catalog.mz_columns (id TEXT, name TEXT, position INTEGER, type_oid INTEGER, type_mod INTEGER, nullable BOOLEAN, "default" TEXT)')
|
|
31
|
+
connection.exec_driver_sql('INSERT INTO mz_catalog.mz_columns VALUES (?, ?, ?, ?, ?, ?, ?)', [
|
|
32
|
+
("u1", "amount", 2, 1700, 655366, True, "0"),
|
|
33
|
+
("u1", "id", 1, 23, -1, False, None),
|
|
34
|
+
("u2", "names", 1, 1015, 24, True, None),
|
|
35
|
+
("unselected", "ignored", 1, 23, -1, False, None),
|
|
36
|
+
])
|
|
37
|
+
result = dict(dialect.get_multi_columns(connection, None, None, ObjectScope.ANY, ObjectKind.ANY))
|
|
38
|
+
assert set(result) == {(None, "users"), (None, "view"), (None, "empty")}
|
|
39
|
+
assert result[(None, "empty")] == []
|
|
40
|
+
columns = result[(None, "users")]
|
|
41
|
+
assert [col["name"] for col in columns] == ["id", "amount"]
|
|
42
|
+
assert isinstance(columns[0]["type"], Integer)
|
|
43
|
+
assert columns[0]["nullable"] is False
|
|
44
|
+
assert columns[0]["default"] is None
|
|
45
|
+
assert isinstance(columns[1]["type"], Numeric)
|
|
46
|
+
assert (columns[1]["type"].precision, columns[1]["type"].scale) == (10, 2)
|
|
47
|
+
assert columns[1]["nullable"] is True
|
|
48
|
+
assert columns[1]["default"] == "0"
|
|
49
|
+
array = result[(None, "view")][0]["type"]
|
|
50
|
+
assert isinstance(array, ARRAY)
|
|
51
|
+
assert isinstance(array.item_type, String)
|
|
52
|
+
assert array.item_type.length == 20
|
|
53
|
+
# The inherited single-relation wrapper also uses this implementation.
|
|
54
|
+
assert dialect.get_columns(connection, "users")[0]["name"] == "id"
|
|
55
|
+
connection.exec_driver_sql("UPDATE mz_catalog.mz_columns SET type_oid = 99999, type_mod = -1 WHERE id = 'u2'")
|
|
56
|
+
with pytest.warns(SAWarning, match="Did not recognize type"):
|
|
57
|
+
result = dict(dialect.get_multi_columns(connection, None, None, ObjectScope.ANY, ObjectKind.ANY))
|
|
58
|
+
assert isinstance(result[(None, "view")][0]["type"], NullType)
|
|
59
|
+
engine.dispose()
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def test_no_matching_relations_skips_column_query():
|
|
63
|
+
dialect = MaterializeDialect_psycopg2()
|
|
64
|
+
dialect._get_relation_ids = Mock(return_value={})
|
|
65
|
+
connection = Mock()
|
|
66
|
+
assert dict(dialect.get_multi_columns(connection, "public", [], ObjectScope.DEFAULT, ObjectKind.TABLE)) == {}
|
|
67
|
+
connection.execute.assert_not_called()
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from sqlalchemy import create_engine
|
|
2
|
+
from sqlalchemy_materialize.psycopg2 import MaterializeDialect_psycopg2
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def test_dialect_registration():
|
|
6
|
+
engine = create_engine(
|
|
7
|
+
"materialize+psycopg2://user:password@localhost:6875/materialize"
|
|
8
|
+
)
|
|
9
|
+
try:
|
|
10
|
+
assert isinstance(engine.dialect, MaterializeDialect_psycopg2)
|
|
11
|
+
assert engine.dialect.name == "materialize"
|
|
12
|
+
assert engine.dialect.driver == "psycopg2"
|
|
13
|
+
finally:
|
|
14
|
+
engine.dispose()
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
"""Index reflection scope tests against a fake mz_catalog.
|
|
2
|
+
|
|
3
|
+
The fixture mirrors catalog behavior observed on Materialize v26.51.1:
|
|
4
|
+
|
|
5
|
+
* Temporary objects have ``schema_id = '0'``. There is no ``mz_temp`` row in
|
|
6
|
+
``mz_schemas``, so joins to mz_schemas / mz_databases yield NULL for them.
|
|
7
|
+
* ``pg_table_is_visible(oid)`` is NULL for temporary objects, and TRUE for a
|
|
8
|
+
search-path object even when a temporary object with the same name shadows it.
|
|
9
|
+
* Temporary objects of *every* session appear in the catalog and cannot be
|
|
10
|
+
told apart, so "other session" temp objects are not modelled here.
|
|
11
|
+
* Indexes cannot be created on temporary objects ("non-temporary items cannot
|
|
12
|
+
depend on temporary item"), so temporary relations never have indexes.
|
|
13
|
+
* In ``mz_index_columns`` exactly one of ``on_position`` / ``on_expression`` is
|
|
14
|
+
non-NULL; positions are 1-based.
|
|
15
|
+
"""
|
|
16
|
+
import pytest
|
|
17
|
+
from sqlalchemy import create_engine
|
|
18
|
+
from sqlalchemy.engine.reflection import ObjectKind, ObjectScope
|
|
19
|
+
|
|
20
|
+
from sqlalchemy_materialize.psycopg2 import MaterializeDialect_psycopg2
|
|
21
|
+
|
|
22
|
+
TEMP = "0"
|
|
23
|
+
|
|
24
|
+
DATABASES = [("u1", "materialize"), ("u2", "other_db")]
|
|
25
|
+
|
|
26
|
+
SCHEMAS = [
|
|
27
|
+
# id, database_id, name
|
|
28
|
+
("u3", "u1", "public"),
|
|
29
|
+
("u4", "u1", "analytics"), # not on the search path
|
|
30
|
+
("u5", "u2", "public"), # public schema of a different database
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
RELATIONS = [
|
|
34
|
+
# id, oid, name, schema_id, catalog, type, visible
|
|
35
|
+
("u1", 20201, "shared", "u3", "mz_tables", "table", True),
|
|
36
|
+
("u3", 20203, "regular", "u3", "mz_tables", "table", True),
|
|
37
|
+
("u5", 20205, "v", "u3", "mz_views", "view", True),
|
|
38
|
+
("u7", 20207, "mv", "u3", "mz_materialized_views", "materialized-view", True),
|
|
39
|
+
("u8", 20208, "report", "u4", "mz_tables", "table", False),
|
|
40
|
+
("u10", 20210, "elsewhere", "u5", "mz_tables", "table", False),
|
|
41
|
+
("u11", 20211, "a_only", TEMP, "mz_tables", "table", None),
|
|
42
|
+
("u12", 20212, "shared", TEMP, "mz_tables", "table", None), # shadows public.shared
|
|
43
|
+
("u14", 20214, "temp_view", TEMP, "mz_views", "view", None),
|
|
44
|
+
]
|
|
45
|
+
|
|
46
|
+
COLUMNS = [
|
|
47
|
+
# relation id, position, name
|
|
48
|
+
("u1", 1, "value"),
|
|
49
|
+
("u3", 1, "a"),
|
|
50
|
+
("u3", 2, "b"),
|
|
51
|
+
("u5", 1, "x"),
|
|
52
|
+
("u7", 1, "x"),
|
|
53
|
+
("u8", 1, "value"),
|
|
54
|
+
("u10", 1, "value"),
|
|
55
|
+
("u11", 1, "value"),
|
|
56
|
+
("u12", 1, "value"),
|
|
57
|
+
("u14", 1, "x"),
|
|
58
|
+
]
|
|
59
|
+
|
|
60
|
+
INDEXES = [
|
|
61
|
+
# id, name, on_id, [(index_position, on_position, on_expression)]
|
|
62
|
+
("u2", "idx_shared", "u1", [(1, 1, None)]),
|
|
63
|
+
("u4", "idx_multi", "u3", [(1, 1, None), (2, None, "b + 1")]),
|
|
64
|
+
("u6", "idx_v", "u5", [(1, 1, None)]),
|
|
65
|
+
("u9", "idx_report", "u8", [(1, 1, None)]),
|
|
66
|
+
("u13", "idx_elsewhere", "u10", [(1, 1, None)]),
|
|
67
|
+
]
|
|
68
|
+
|
|
69
|
+
# Expected reflected indexes per relation name.
|
|
70
|
+
IDX_SHARED = {"name": "idx_shared", "column_names": ["value"], "unique": False}
|
|
71
|
+
IDX_MULTI = {
|
|
72
|
+
"name": "idx_multi",
|
|
73
|
+
"column_names": ["a", None],
|
|
74
|
+
"expressions": ["a", "b + 1"],
|
|
75
|
+
"unique": False,
|
|
76
|
+
}
|
|
77
|
+
IDX_V = {"name": "idx_v", "column_names": ["x"], "unique": False}
|
|
78
|
+
IDX_REPORT = {"name": "idx_report", "column_names": ["value"], "unique": False}
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@pytest.fixture
|
|
82
|
+
def catalog_connection():
|
|
83
|
+
engine = create_engine("sqlite://")
|
|
84
|
+
visibility = {oid: visible for _, oid, *_, visible in RELATIONS}
|
|
85
|
+
with engine.connect() as connection:
|
|
86
|
+
dbapi = connection.connection.driver_connection
|
|
87
|
+
dbapi.create_function("current_database", 0, lambda: "materialize")
|
|
88
|
+
dbapi.create_function("pg_table_is_visible", 1, visibility.get)
|
|
89
|
+
connection.exec_driver_sql("ATTACH DATABASE ':memory:' AS mz_catalog")
|
|
90
|
+
relation_cols = "id TEXT, oid INTEGER, schema_id TEXT, name TEXT"
|
|
91
|
+
definitions = {
|
|
92
|
+
"mz_databases": "id TEXT, oid INTEGER, name TEXT",
|
|
93
|
+
"mz_schemas": "id TEXT, oid INTEGER, database_id TEXT, name TEXT",
|
|
94
|
+
"mz_tables": relation_cols,
|
|
95
|
+
"mz_views": relation_cols,
|
|
96
|
+
"mz_materialized_views": relation_cols,
|
|
97
|
+
"mz_relations": relation_cols + ", type TEXT",
|
|
98
|
+
"mz_columns": "id TEXT, name TEXT, position INTEGER",
|
|
99
|
+
"mz_indexes": "id TEXT, oid INTEGER, name TEXT, on_id TEXT",
|
|
100
|
+
"mz_index_columns": (
|
|
101
|
+
"index_id TEXT, index_position INTEGER, "
|
|
102
|
+
"on_position INTEGER, on_expression TEXT"
|
|
103
|
+
),
|
|
104
|
+
}
|
|
105
|
+
for table, columns in definitions.items():
|
|
106
|
+
connection.exec_driver_sql(f"CREATE TABLE mz_catalog.{table} ({columns})")
|
|
107
|
+
|
|
108
|
+
def insert(table, *values):
|
|
109
|
+
marks = ", ".join("?" * len(values))
|
|
110
|
+
connection.exec_driver_sql(
|
|
111
|
+
f"INSERT INTO mz_catalog.{table} VALUES ({marks})", values
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
for n, (db_id, name) in enumerate(DATABASES):
|
|
115
|
+
insert("mz_databases", db_id, 20000 + n, name)
|
|
116
|
+
for n, (schema_id, db_id, name) in enumerate(SCHEMAS):
|
|
117
|
+
insert("mz_schemas", schema_id, 20100 + n, db_id, name)
|
|
118
|
+
for rel_id, oid, name, schema_id, catalog, rel_type, _ in RELATIONS:
|
|
119
|
+
insert(catalog, rel_id, oid, schema_id, name)
|
|
120
|
+
insert("mz_relations", rel_id, oid, schema_id, name, rel_type)
|
|
121
|
+
for rel_id, position, name in COLUMNS:
|
|
122
|
+
insert("mz_columns", rel_id, name, position)
|
|
123
|
+
for n, (index_id, name, on_id, columns) in enumerate(INDEXES):
|
|
124
|
+
insert("mz_indexes", index_id, 20300 + n, name, on_id)
|
|
125
|
+
for column in columns:
|
|
126
|
+
insert("mz_index_columns", index_id, *column)
|
|
127
|
+
yield connection
|
|
128
|
+
engine.dispose()
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
@pytest.mark.parametrize("schema,scope,kind,filters,expected", [
|
|
132
|
+
# Search path (schema=None), regular objects only.
|
|
133
|
+
(None, ObjectScope.DEFAULT, ObjectKind.ANY, None,
|
|
134
|
+
{"shared": [IDX_SHARED], "regular": [IDX_MULTI], "v": [IDX_V], "mv": []}),
|
|
135
|
+
(None, ObjectScope.DEFAULT, ObjectKind.TABLE, None,
|
|
136
|
+
{"shared": [IDX_SHARED], "regular": [IDX_MULTI]}),
|
|
137
|
+
(None, ObjectScope.DEFAULT, ObjectKind.VIEW, None, {"v": [IDX_V]}),
|
|
138
|
+
(None, ObjectScope.DEFAULT, ObjectKind.MATERIALIZED_VIEW, None, {"mv": []}),
|
|
139
|
+
(None, ObjectScope.DEFAULT, ObjectKind.TABLE, ["shared", "missing"],
|
|
140
|
+
{"shared": [IDX_SHARED]}),
|
|
141
|
+
# Temporary objects: always present, never indexed.
|
|
142
|
+
(None, ObjectScope.TEMPORARY, ObjectKind.ANY, None,
|
|
143
|
+
{"a_only": [], "shared": [], "temp_view": []}),
|
|
144
|
+
(None, ObjectScope.TEMPORARY, ObjectKind.TABLE, None,
|
|
145
|
+
{"a_only": [], "shared": []}),
|
|
146
|
+
(None, ObjectScope.TEMPORARY, ObjectKind.MATERIALIZED_VIEW, None, {}),
|
|
147
|
+
# Both scopes: the temporary "shared" shadows public.shared, so the
|
|
148
|
+
# unqualified name resolves to the (unindexed) temporary table.
|
|
149
|
+
(None, ObjectScope.ANY, ObjectKind.ANY, None,
|
|
150
|
+
{"shared": [], "regular": [IDX_MULTI], "v": [IDX_V], "mv": [],
|
|
151
|
+
"a_only": [], "temp_view": []}),
|
|
152
|
+
# Explicit schemas.
|
|
153
|
+
("public", ObjectScope.DEFAULT, ObjectKind.ANY, None,
|
|
154
|
+
{"shared": [IDX_SHARED], "regular": [IDX_MULTI], "v": [IDX_V], "mv": []}),
|
|
155
|
+
("public", ObjectScope.TEMPORARY, ObjectKind.ANY, None, {}),
|
|
156
|
+
("analytics", ObjectScope.DEFAULT, ObjectKind.TABLE, None,
|
|
157
|
+
{"report": [IDX_REPORT]}),
|
|
158
|
+
("mz_temp", ObjectScope.ANY, ObjectKind.TABLE, ["shared", "a_only", "missing"],
|
|
159
|
+
{"shared": [], "a_only": []}),
|
|
160
|
+
("mz_temp", ObjectScope.DEFAULT, ObjectKind.ANY, None, {}),
|
|
161
|
+
# Empty filter list.
|
|
162
|
+
(None, ObjectScope.ANY, ObjectKind.ANY, [], {}),
|
|
163
|
+
])
|
|
164
|
+
def test_index_scope(catalog_connection, schema, scope, kind, filters, expected):
|
|
165
|
+
dialect = MaterializeDialect_psycopg2()
|
|
166
|
+
result = dict(dialect.get_multi_indexes(catalog_connection, schema, filters, scope, kind))
|
|
167
|
+
assert result == {(schema, name): indexes for name, indexes in expected.items()}
|