prostgles-server 4.2.159 → 4.2.161
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Auth/setEmailProvider.js +2 -2
- package/dist/Auth/setEmailProvider.js.map +1 -1
- package/lib/Auth/AuthHandler.ts +436 -0
- package/lib/Auth/AuthTypes.ts +280 -0
- package/lib/Auth/getSafeReturnURL.ts +35 -0
- package/lib/Auth/sendEmail.ts +83 -0
- package/lib/Auth/setAuthProviders.ts +128 -0
- package/lib/Auth/setEmailProvider.ts +85 -0
- package/lib/Auth/setupAuthRoutes.ts +161 -0
- package/lib/DBEventsManager.ts +178 -0
- package/lib/DBSchemaBuilder.ts +225 -0
- package/lib/DboBuilder/DboBuilder.ts +319 -0
- package/lib/DboBuilder/DboBuilderTypes.ts +361 -0
- package/lib/DboBuilder/QueryBuilder/Functions.ts +1153 -0
- package/lib/DboBuilder/QueryBuilder/QueryBuilder.ts +288 -0
- package/lib/DboBuilder/QueryBuilder/getJoinQuery.ts +263 -0
- package/lib/DboBuilder/QueryBuilder/getNewQuery.ts +271 -0
- package/lib/DboBuilder/QueryBuilder/getSelectQuery.ts +136 -0
- package/lib/DboBuilder/QueryBuilder/prepareHaving.ts +22 -0
- package/lib/DboBuilder/QueryStreamer.ts +250 -0
- package/lib/DboBuilder/TableHandler/DataValidator.ts +428 -0
- package/lib/DboBuilder/TableHandler/TableHandler.ts +205 -0
- package/lib/DboBuilder/TableHandler/delete.ts +115 -0
- package/lib/DboBuilder/TableHandler/insert.ts +183 -0
- package/lib/DboBuilder/TableHandler/insertTest.ts +78 -0
- package/lib/DboBuilder/TableHandler/onDeleteFromFileTable.ts +62 -0
- package/lib/DboBuilder/TableHandler/runInsertUpdateQuery.ts +134 -0
- package/lib/DboBuilder/TableHandler/update.ts +126 -0
- package/lib/DboBuilder/TableHandler/updateBatch.ts +49 -0
- package/lib/DboBuilder/TableHandler/updateFile.ts +48 -0
- package/lib/DboBuilder/TableHandler/upsert.ts +34 -0
- package/lib/DboBuilder/ViewHandler/ViewHandler.ts +393 -0
- package/lib/DboBuilder/ViewHandler/count.ts +38 -0
- package/lib/DboBuilder/ViewHandler/find.ts +153 -0
- package/lib/DboBuilder/ViewHandler/getExistsCondition.ts +73 -0
- package/lib/DboBuilder/ViewHandler/getExistsFilters.ts +74 -0
- package/lib/DboBuilder/ViewHandler/getInfo.ts +32 -0
- package/lib/DboBuilder/ViewHandler/getTableJoinQuery.ts +84 -0
- package/lib/DboBuilder/ViewHandler/parseComplexFilter.ts +96 -0
- package/lib/DboBuilder/ViewHandler/parseFieldFilter.ts +105 -0
- package/lib/DboBuilder/ViewHandler/parseJoinPath.ts +208 -0
- package/lib/DboBuilder/ViewHandler/prepareSortItems.ts +163 -0
- package/lib/DboBuilder/ViewHandler/prepareWhere.ts +90 -0
- package/lib/DboBuilder/ViewHandler/size.ts +37 -0
- package/lib/DboBuilder/ViewHandler/subscribe.ts +118 -0
- package/lib/DboBuilder/ViewHandler/validateViewRules.ts +70 -0
- package/lib/DboBuilder/dboBuilderUtils.ts +222 -0
- package/lib/DboBuilder/getColumns.ts +114 -0
- package/lib/DboBuilder/getCondition.ts +201 -0
- package/lib/DboBuilder/getSubscribeRelatedTables.ts +190 -0
- package/lib/DboBuilder/getTablesForSchemaPostgresSQL.ts +426 -0
- package/lib/DboBuilder/insertNestedRecords.ts +355 -0
- package/lib/DboBuilder/parseUpdateRules.ts +187 -0
- package/lib/DboBuilder/prepareShortestJoinPaths.ts +186 -0
- package/lib/DboBuilder/runSQL.ts +182 -0
- package/lib/DboBuilder/runTransaction.ts +50 -0
- package/lib/DboBuilder/sqlErrCodeToMsg.ts +254 -0
- package/lib/DboBuilder/uploadFile.ts +69 -0
- package/lib/Event_Trigger_Tags.ts +118 -0
- package/lib/FileManager/FileManager.ts +358 -0
- package/lib/FileManager/getValidatedFileType.ts +69 -0
- package/lib/FileManager/initFileManager.ts +187 -0
- package/lib/FileManager/upload.ts +62 -0
- package/lib/FileManager/uploadStream.ts +79 -0
- package/lib/Filtering.ts +463 -0
- package/lib/JSONBValidation/validate_jsonb_schema_sql.ts +502 -0
- package/lib/JSONBValidation/validation.ts +143 -0
- package/lib/Logging.ts +127 -0
- package/lib/PostgresNotifListenManager.ts +143 -0
- package/lib/Prostgles.ts +485 -0
- package/lib/ProstglesTypes.ts +196 -0
- package/lib/PubSubManager/PubSubManager.ts +609 -0
- package/lib/PubSubManager/addSub.ts +138 -0
- package/lib/PubSubManager/addSync.ts +141 -0
- package/lib/PubSubManager/getCreatePubSubManagerError.ts +72 -0
- package/lib/PubSubManager/getPubSubManagerInitQuery.ts +662 -0
- package/lib/PubSubManager/initPubSubManager.ts +79 -0
- package/lib/PubSubManager/notifListener.ts +173 -0
- package/lib/PubSubManager/orphanTriggerCheck.ts +70 -0
- package/lib/PubSubManager/pushSubData.ts +55 -0
- package/lib/PublishParser/PublishParser.ts +162 -0
- package/lib/PublishParser/getFileTableRules.ts +124 -0
- package/lib/PublishParser/getSchemaFromPublish.ts +141 -0
- package/lib/PublishParser/getTableRulesWithoutFileTable.ts +177 -0
- package/lib/PublishParser/publishTypesAndUtils.ts +399 -0
- package/lib/RestApi.ts +127 -0
- package/lib/SchemaWatch/SchemaWatch.ts +90 -0
- package/lib/SchemaWatch/createSchemaWatchEventTrigger.ts +3 -0
- package/lib/SchemaWatch/getValidatedWatchSchemaType.ts +45 -0
- package/lib/SchemaWatch/getWatchSchemaTagList.ts +27 -0
- package/lib/SyncReplication.ts +557 -0
- package/lib/TableConfig/TableConfig.ts +468 -0
- package/lib/TableConfig/getColumnDefinitionQuery.ts +111 -0
- package/lib/TableConfig/getConstraintDefinitionQueries.ts +95 -0
- package/lib/TableConfig/getFutureTableSchema.ts +64 -0
- package/lib/TableConfig/getPGIndexes.ts +53 -0
- package/lib/TableConfig/getTableColumnQueries.ts +129 -0
- package/lib/TableConfig/initTableConfig.ts +326 -0
- package/lib/index.ts +13 -0
- package/lib/initProstgles.ts +319 -0
- package/lib/onSocketConnected.ts +102 -0
- package/lib/runClientRequest.ts +129 -0
- package/lib/shortestPath.ts +122 -0
- package/lib/typeTests/DBoGenerated.d.ts +320 -0
- package/lib/typeTests/dboTypeCheck.ts +81 -0
- package/lib/utils.ts +15 -0
- package/package.json +1 -1
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
import { AnyObject, asName, ParsedJoinPath, reverseParsedPath, SubscribeParams } from "prostgles-types";
|
|
2
|
+
import { TableRule } from "../PublishParser/PublishParser";
|
|
3
|
+
import { log, ViewSubscriptionOptions } from "../PubSubManager/PubSubManager";
|
|
4
|
+
import { Filter, getSerializedClientErrorFromPGError, LocalParams } from "./DboBuilder";
|
|
5
|
+
import { NewQuery } from "./QueryBuilder/QueryBuilder";
|
|
6
|
+
import { ViewHandler } from "./ViewHandler/ViewHandler";
|
|
7
|
+
|
|
8
|
+
type Args = {
|
|
9
|
+
selectParams: Omit<SubscribeParams, "throttle">;
|
|
10
|
+
filter: Filter;
|
|
11
|
+
table_rules: TableRule<AnyObject, void> | undefined;
|
|
12
|
+
localParams: LocalParams | undefined;
|
|
13
|
+
newQuery: NewQuery;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* When subscribing to a view: we identify underlying tables to subscribe to them
|
|
18
|
+
* When subscribing to a table: we identify joined tables to subscribe to them
|
|
19
|
+
*/
|
|
20
|
+
export async function getSubscribeRelatedTables(this: ViewHandler, { filter, localParams, newQuery }: Args){
|
|
21
|
+
|
|
22
|
+
let viewOptions: ViewSubscriptionOptions | undefined = undefined;
|
|
23
|
+
const { condition } = newQuery.whereOpts;
|
|
24
|
+
if (this.is_view) {
|
|
25
|
+
/** TODO: this needs to be memoized on schema fetch */
|
|
26
|
+
const viewName = this.name;
|
|
27
|
+
const viewNameEscaped = this.escapedName;
|
|
28
|
+
const { current_schema } = await this.db.oneOrNone("SELECT current_schema")
|
|
29
|
+
|
|
30
|
+
/** Get list of used columns and their parent tables */
|
|
31
|
+
let { def } = (await this.db.oneOrNone("SELECT pg_get_viewdef(${viewName}) as def", { viewName })) as { def: string };
|
|
32
|
+
def = def.trim();
|
|
33
|
+
if (def.endsWith(";")) {
|
|
34
|
+
def = def.slice(0, -1);
|
|
35
|
+
}
|
|
36
|
+
if (!def || typeof def !== "string") {
|
|
37
|
+
throw getSerializedClientErrorFromPGError("Could get view definition", { type: "tableMethod", localParams, view: this, });
|
|
38
|
+
}
|
|
39
|
+
const { fields } = await this.dboBuilder.dbo.sql!(`SELECT * FROM ( \n ${def} \n ) prostgles_subscribe_view_definition LIMIT 0`, {});
|
|
40
|
+
const tableColumns = fields.filter(f => f.tableName && f.columnName);
|
|
41
|
+
|
|
42
|
+
/** Create exists filters for each table */
|
|
43
|
+
const tableIds: string[] = Array.from(new Set(tableColumns.map(tc => tc.tableID!.toString())));
|
|
44
|
+
viewOptions = {
|
|
45
|
+
type: "view",
|
|
46
|
+
viewName,
|
|
47
|
+
definition: def,
|
|
48
|
+
relatedTables: []
|
|
49
|
+
}
|
|
50
|
+
viewOptions.relatedTables = await Promise.all(tableIds.map(async tableID => {
|
|
51
|
+
const table = this.dboBuilder.USER_TABLES!.find(t => t.relid === +tableID)!;
|
|
52
|
+
let tableCols = tableColumns.filter(tc => tc.tableID!.toString() === tableID);
|
|
53
|
+
|
|
54
|
+
/** If table has primary keys and they are all in this view then use only primary keys */
|
|
55
|
+
if (table?.pkey_columns?.every(pkey => tableCols.some(c => c.columnName === pkey))) {
|
|
56
|
+
tableCols = tableCols.filter(c => table?.pkey_columns?.includes(c.columnName!))
|
|
57
|
+
} else {
|
|
58
|
+
/** Exclude non comparable data types */
|
|
59
|
+
tableCols = tableCols.filter(c => !["json", "xml"].includes(c.udt_name))
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const { relname: tableName, schemaname: tableSchema } = table;
|
|
63
|
+
|
|
64
|
+
if (tableCols.length) {
|
|
65
|
+
|
|
66
|
+
const tableNameEscaped = tableSchema === current_schema ? table.relname : [tableSchema, tableName].map(v => JSON.stringify(v)).join(".");
|
|
67
|
+
|
|
68
|
+
const fullCondition = `EXISTS (
|
|
69
|
+
SELECT 1
|
|
70
|
+
FROM ${viewNameEscaped}
|
|
71
|
+
WHERE ${tableCols.map(c => `${tableNameEscaped}.${JSON.stringify(c.columnName)} = ${viewNameEscaped}.${JSON.stringify(c.name)}`).join(" AND \n")}
|
|
72
|
+
AND ${condition || "TRUE"}
|
|
73
|
+
)`;
|
|
74
|
+
|
|
75
|
+
try {
|
|
76
|
+
const { count } = await this.db.oneOrNone(`
|
|
77
|
+
WITH ${asName(tableName)} AS (
|
|
78
|
+
SELECT *
|
|
79
|
+
FROM ${asName(tableName)}
|
|
80
|
+
LIMIT 0
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
SELECT COUNT(*) as count
|
|
84
|
+
FROM (
|
|
85
|
+
${def}
|
|
86
|
+
) prostgles_view_ref_table_test
|
|
87
|
+
`);
|
|
88
|
+
|
|
89
|
+
const relatedTableSubscription = {
|
|
90
|
+
tableName: tableName!,
|
|
91
|
+
tableNameEscaped,
|
|
92
|
+
condition: fullCondition,
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
if (count.toString() === '0') {
|
|
96
|
+
return relatedTableSubscription;
|
|
97
|
+
}
|
|
98
|
+
} catch (e) {
|
|
99
|
+
log(`Could not not override subscribed view (${this.name}) table (${tableName}). Will not check condition`, e);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
return {
|
|
104
|
+
tableName,
|
|
105
|
+
tableNameEscaped: JSON.stringify(tableName),// [table.schemaname, table.relname].map(v => JSON.stringify(v)).join("."),
|
|
106
|
+
condition: "TRUE"
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
}))
|
|
110
|
+
|
|
111
|
+
/** Get list of remaining used inner tables */
|
|
112
|
+
const allUsedTables: { table_name: string; table_schema: string; }[] = await this.db.any(
|
|
113
|
+
"SELECT distinct table_name, table_schema FROM information_schema.view_column_usage WHERE view_name = ${viewName}",
|
|
114
|
+
{ viewName }
|
|
115
|
+
);
|
|
116
|
+
|
|
117
|
+
/** Remaining tables will have listeners on all records (condition = "TRUE") */
|
|
118
|
+
const remainingInnerTables = allUsedTables.filter(at => !tableColumns.some(dc => dc.tableName === at.table_name && dc.tableSchema === at.table_schema));
|
|
119
|
+
viewOptions.relatedTables = [
|
|
120
|
+
...viewOptions.relatedTables,
|
|
121
|
+
...remainingInnerTables.map(t => ({
|
|
122
|
+
tableName: t.table_name,
|
|
123
|
+
tableNameEscaped: [t.table_name, t.table_schema].map(v => JSON.stringify(v)).join("."),
|
|
124
|
+
condition: "TRUE"
|
|
125
|
+
}))
|
|
126
|
+
];
|
|
127
|
+
|
|
128
|
+
if (!viewOptions.relatedTables.length) {
|
|
129
|
+
throw "Could not subscribe to this view: no related tables found";
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** Any joined table used within select or filter must also be added a trigger for this recordset */
|
|
133
|
+
} else {
|
|
134
|
+
viewOptions = {
|
|
135
|
+
type: "table",
|
|
136
|
+
relatedTables: []
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
const nonExistsFilter = newQuery.whereOpts.exists.length ? {} : filter;
|
|
140
|
+
const pushRelatedTable = async (relatedTableName: string, joinPath: ParsedJoinPath[]) => {
|
|
141
|
+
const relatedTableOrViewHandler = this.dboBuilder.dbo[relatedTableName];
|
|
142
|
+
if (!relatedTableOrViewHandler) {
|
|
143
|
+
throw `Table ${relatedTableName} not found`;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
const alreadyPushed = viewOptions?.relatedTables.find(rt => rt.tableName === relatedTableName)
|
|
147
|
+
if(alreadyPushed || relatedTableOrViewHandler.is_view){
|
|
148
|
+
return
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
viewOptions ??= {
|
|
152
|
+
type: "table",
|
|
153
|
+
relatedTables: []
|
|
154
|
+
}
|
|
155
|
+
viewOptions.relatedTables.push({
|
|
156
|
+
tableName: relatedTableName,
|
|
157
|
+
tableNameEscaped: asName(relatedTableName),
|
|
158
|
+
condition: (await relatedTableOrViewHandler!.prepareWhere!({
|
|
159
|
+
select: undefined,
|
|
160
|
+
filter: {
|
|
161
|
+
$existsJoined: {
|
|
162
|
+
path: reverseParsedPath(joinPath, this.name),
|
|
163
|
+
filter: nonExistsFilter
|
|
164
|
+
}
|
|
165
|
+
},
|
|
166
|
+
addWhere: false,
|
|
167
|
+
localParams: undefined,
|
|
168
|
+
tableRule: undefined
|
|
169
|
+
})).where
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* Avoid nested exists error. Will affect performance
|
|
175
|
+
*/
|
|
176
|
+
for await (const j of (newQuery.joins ?? [])) {
|
|
177
|
+
await pushRelatedTable(j.table, j.joinPath);
|
|
178
|
+
}
|
|
179
|
+
for await (const e of newQuery.whereOpts.exists.filter(e => e.isJoined)) {
|
|
180
|
+
if(!e.isJoined) throw `Not possible`;
|
|
181
|
+
const targetTable = e.parsedPath.at(-1)!.table;
|
|
182
|
+
await pushRelatedTable(targetTable, e.parsedPath);
|
|
183
|
+
}
|
|
184
|
+
if (!viewOptions.relatedTables.length) {
|
|
185
|
+
viewOptions = undefined;
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
return viewOptions;
|
|
190
|
+
}
|
|
@@ -0,0 +1,426 @@
|
|
|
1
|
+
import { SQLResult, asName } from "prostgles-types";
|
|
2
|
+
import { omitKeys, tryCatch } from "prostgles-types/dist/util";
|
|
3
|
+
import { DboBuilder } from "../DboBuilder/DboBuilder";
|
|
4
|
+
import { DBorTx } from "../Prostgles";
|
|
5
|
+
import { clone } from "../utils";
|
|
6
|
+
import { TableSchema, TableSchemaColumn } from "./DboBuilderTypes";
|
|
7
|
+
import { ProstglesInitOptions } from "../ProstglesTypes";
|
|
8
|
+
|
|
9
|
+
const getMaterialViews = (db: DBorTx, schema: ProstglesInitOptions["schema"]) => {
|
|
10
|
+
const { sql, schemaNames } = getSchemaFilter(schema);
|
|
11
|
+
|
|
12
|
+
const query = `
|
|
13
|
+
SELECT
|
|
14
|
+
c.oid,
|
|
15
|
+
schema,
|
|
16
|
+
escaped_identifier,
|
|
17
|
+
true as is_view,
|
|
18
|
+
true as is_mat_view,
|
|
19
|
+
obj_description(c.oid) as comment,
|
|
20
|
+
c.table_name as name,
|
|
21
|
+
definition as view_definition,
|
|
22
|
+
jsonb_build_object(
|
|
23
|
+
'insert', FALSE,
|
|
24
|
+
'select', TRUE,
|
|
25
|
+
'update', FALSE,
|
|
26
|
+
'delete', FALSE
|
|
27
|
+
) as privileges,
|
|
28
|
+
json_agg(json_build_object(
|
|
29
|
+
'name', column_name,
|
|
30
|
+
'table_oid', c.oid,
|
|
31
|
+
'is_pkey', false,
|
|
32
|
+
'data_type', data_type,
|
|
33
|
+
'udt_name', udt_name,
|
|
34
|
+
'element_udt_name',
|
|
35
|
+
CASE WHEN LEFT(udt_name, 1) = '_'
|
|
36
|
+
THEN RIGHT(udt_name, -1) END,
|
|
37
|
+
'element_type',
|
|
38
|
+
CASE WHEN RIGHT(data_type, 2) = '[]'
|
|
39
|
+
THEN LEFT(data_type, -2) END,
|
|
40
|
+
'is_nullable', nullable,
|
|
41
|
+
'is_generated', true,
|
|
42
|
+
'references', null,
|
|
43
|
+
'has_default', false,
|
|
44
|
+
'column_default', null,
|
|
45
|
+
'is_updatable', false,
|
|
46
|
+
'privileges', $$ { "SELECT": true } $$::jsonb
|
|
47
|
+
)) as columns
|
|
48
|
+
FROM pg_catalog.pg_matviews m
|
|
49
|
+
INNER JOIN (
|
|
50
|
+
SELECT
|
|
51
|
+
t.oid,
|
|
52
|
+
CASE WHEN current_schema() = s.nspname
|
|
53
|
+
THEN format('%I', t.relname)
|
|
54
|
+
ELSE format('%I.%I', s.nspname, t.relname)
|
|
55
|
+
END as escaped_identifier,
|
|
56
|
+
t.relname as table_name,
|
|
57
|
+
s.nspname as schema,
|
|
58
|
+
a.attname as column_name,
|
|
59
|
+
pg_catalog.format_type(a.atttypid, a.atttypmod) as data_type,
|
|
60
|
+
typname as udt_name,
|
|
61
|
+
a.attnotnull as nullable,
|
|
62
|
+
a.attnum as ordinal_position,
|
|
63
|
+
col_description(t.oid, attnum) as comment
|
|
64
|
+
FROM pg_catalog.pg_attribute a
|
|
65
|
+
JOIN pg_catalog.pg_class t on a.attrelid = t.oid
|
|
66
|
+
JOIN pg_catalog.pg_namespace s on t.relnamespace = s.oid
|
|
67
|
+
JOIN pg_catalog.pg_type pt ON pt.oid = a.atttypid
|
|
68
|
+
WHERE a.attnum > 0
|
|
69
|
+
AND NOT a.attisdropped
|
|
70
|
+
AND relkind = 'm'
|
|
71
|
+
ORDER BY a.attnum
|
|
72
|
+
) c
|
|
73
|
+
ON matviewname = table_name
|
|
74
|
+
AND schemaname = schema
|
|
75
|
+
WHERE schema ${sql}
|
|
76
|
+
GROUP BY c.oid, escaped_identifier, c.table_name, schema, definition
|
|
77
|
+
`;
|
|
78
|
+
|
|
79
|
+
/** TODO: check privileges
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
select
|
|
84
|
+
coalesce(nullif(s[1], ''), 'public') as grantee,
|
|
85
|
+
s[2] as privileges
|
|
86
|
+
from
|
|
87
|
+
pg_class c
|
|
88
|
+
join pg_namespace n on n.oid = relnamespace
|
|
89
|
+
join pg_roles r on r.oid = relowner,
|
|
90
|
+
unnest(coalesce(relacl::text[], format('{%s=arwdDxt/%s}', rolname, rolname)::text[])) acl,
|
|
91
|
+
regexp_split_to_array(acl, '=|/') s
|
|
92
|
+
where nspname = 'public' and relname = 'test_view';
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
*/
|
|
96
|
+
|
|
97
|
+
return db.any(query, { schemaNames });
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export const getSchemaFilter = (schema: ProstglesInitOptions["schema"] = { public: 1 }) => {
|
|
101
|
+
const schemaNames = Object.keys(schema);
|
|
102
|
+
const isInclusive = Object.values(schema).every(v => v);
|
|
103
|
+
if(!schemaNames.length){
|
|
104
|
+
throw "Must specify at least one schema";
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
return {
|
|
108
|
+
sql: ` ${isInclusive? "" : "NOT "}IN (\${schemaNames:csv})`,
|
|
109
|
+
schemaNames,
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
// TODO: Add a onSocketConnect timeout for this query.
|
|
114
|
+
// Reason: this query gets blocked by prostgles.app_triggers from PubSubManager.addTrigger in some cases (pg_dump locks that table)
|
|
115
|
+
export async function getTablesForSchemaPostgresSQL(
|
|
116
|
+
{ db, runSQL }: DboBuilder,
|
|
117
|
+
schema: ProstglesInitOptions["schema"]
|
|
118
|
+
): Promise<{
|
|
119
|
+
result: TableSchema[];
|
|
120
|
+
durations: Record<string, number>;
|
|
121
|
+
}> {
|
|
122
|
+
const { sql, schemaNames } = getSchemaFilter(schema);
|
|
123
|
+
|
|
124
|
+
return db.tx(async t => {
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* Multiple queries to reduce load on low power machines
|
|
128
|
+
*/
|
|
129
|
+
const getFkeys = await tryCatch(async () => {
|
|
130
|
+
|
|
131
|
+
const fkeys: {
|
|
132
|
+
oid: number;
|
|
133
|
+
ftable: string;
|
|
134
|
+
cols: string[];
|
|
135
|
+
fcols: string[];
|
|
136
|
+
}[] = await t.any(`
|
|
137
|
+
WITH pg_class_schema AS (
|
|
138
|
+
SELECT c.oid, c.relname, nspname as schema
|
|
139
|
+
,CASE WHEN current_schema() = nspname
|
|
140
|
+
THEN format('%I', c.relname)
|
|
141
|
+
ELSE format('%I.%I', nspname, c.relname)
|
|
142
|
+
END as escaped_identifier
|
|
143
|
+
FROM pg_catalog.pg_class AS c
|
|
144
|
+
LEFT JOIN pg_catalog.pg_namespace AS ns
|
|
145
|
+
ON c.relnamespace = ns.oid
|
|
146
|
+
WHERE nspname ${sql}
|
|
147
|
+
), fk AS (
|
|
148
|
+
SELECT conrelid as oid
|
|
149
|
+
, escaped_identifier as ftable
|
|
150
|
+
, array_agg(DISTINCT c1.attname::text) as cols
|
|
151
|
+
, array_agg(DISTINCT c2.attname::text) as fcols
|
|
152
|
+
FROM pg_catalog.pg_constraint c
|
|
153
|
+
INNER JOIN pg_class_schema pc
|
|
154
|
+
ON confrelid = pc.oid
|
|
155
|
+
LEFT JOIN pg_attribute c1
|
|
156
|
+
ON c1.attrelid = c.conrelid and ARRAY[c1.attnum] <@ c.conkey
|
|
157
|
+
LEFT JOIN pg_attribute c2
|
|
158
|
+
ON c2.attrelid = c.confrelid and ARRAY[c2.attnum] <@ c.confkey
|
|
159
|
+
WHERE contype = 'f'
|
|
160
|
+
GROUP BY conrelid, conname, pc.escaped_identifier
|
|
161
|
+
)
|
|
162
|
+
SELECT * FROM fk
|
|
163
|
+
`, { schemaNames });
|
|
164
|
+
|
|
165
|
+
return { fkeys };
|
|
166
|
+
});
|
|
167
|
+
if(getFkeys.error !== undefined){
|
|
168
|
+
throw getFkeys.error;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
const badFkey = getFkeys.fkeys!.find(r => r.fcols.includes(null as any));
|
|
172
|
+
if(badFkey){
|
|
173
|
+
throw `Invalid table column schema. Null or empty fcols for ${JSON.stringify(getFkeys.fkeys)}`;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
const getTVColumns = await tryCatch(async () => {
|
|
177
|
+
const columns: (TableSchemaColumn & { table_oid: number; })[] = await t.any(`
|
|
178
|
+
SELECT
|
|
179
|
+
table_oid
|
|
180
|
+
, ccc.column_name as name ,
|
|
181
|
+
ccc.data_type,
|
|
182
|
+
ccc.udt_name,
|
|
183
|
+
ccc.element_type,
|
|
184
|
+
ccc.element_udt_name,
|
|
185
|
+
ccc.is_pkey,
|
|
186
|
+
col_description(table_oid, ordinal_position) as comment,
|
|
187
|
+
ccc.ordinal_position,
|
|
188
|
+
ccc.is_nullable = 'YES' as is_nullable,
|
|
189
|
+
ccc.is_updatable,
|
|
190
|
+
ccc.is_generated,
|
|
191
|
+
null as references,
|
|
192
|
+
ccc.has_default,
|
|
193
|
+
ccc.column_default
|
|
194
|
+
, COALESCE(ccc.privileges, '[]'::JSON) as privileges
|
|
195
|
+
FROM (
|
|
196
|
+
SELECT c.table_schema, c.table_name, c.column_name, c.data_type, c.udt_name
|
|
197
|
+
, e.data_type as element_type
|
|
198
|
+
, e.udt_name as element_udt_name
|
|
199
|
+
, format('%I.%I', c.table_schema, c.table_name)::regclass::oid as table_oid
|
|
200
|
+
--, fc.references
|
|
201
|
+
, c.is_identity = 'YES' OR has_pkey IS TRUE as is_pkey
|
|
202
|
+
, c.ordinal_position
|
|
203
|
+
, COALESCE(c.column_default IS NOT NULL OR c.identity_generation = 'ALWAYS', false) as has_default
|
|
204
|
+
, c.column_default
|
|
205
|
+
, c.is_nullable
|
|
206
|
+
, CASE WHEN c.is_generated = 'ALWAYS' THEN true ELSE false END as is_generated
|
|
207
|
+
/* generated always and view columns cannot be updated */
|
|
208
|
+
, COALESCE(c.is_updatable, 'YES') = 'YES' AND COALESCE(c.is_generated, '') != 'ALWAYS' AND COALESCE(c.identity_generation, '') != 'ALWAYS' as is_updatable
|
|
209
|
+
, cp.privileges
|
|
210
|
+
FROM information_schema.columns c
|
|
211
|
+
LEFT JOIN information_schema.element_types e
|
|
212
|
+
ON ((c.table_catalog, c.table_schema, c.table_name, 'TABLE', c.dtd_identifier)
|
|
213
|
+
= (e.object_catalog, e.object_schema, e.object_name, e.object_type, e.collection_type_identifier)
|
|
214
|
+
)
|
|
215
|
+
LEFT JOIN (
|
|
216
|
+
SELECT DISTINCT tc.table_schema, tc.table_name, kcu.column_name, true as has_pkey
|
|
217
|
+
FROM information_schema.table_constraints as tc
|
|
218
|
+
JOIN information_schema.key_column_usage AS kcu
|
|
219
|
+
ON tc.constraint_name = kcu.constraint_name AND tc.table_schema = kcu.table_schema
|
|
220
|
+
WHERE tc.constraint_type IN ('PRIMARY KEY')
|
|
221
|
+
AND tc.table_schema ${sql}
|
|
222
|
+
) pkeys
|
|
223
|
+
ON
|
|
224
|
+
pkeys.table_schema = c.table_schema
|
|
225
|
+
AND pkeys.table_name = c.table_name
|
|
226
|
+
AND pkeys.column_name = c.column_name
|
|
227
|
+
LEFT JOIN (
|
|
228
|
+
SELECT table_schema, table_name, column_name
|
|
229
|
+
, (json_object_agg(privilege_type, true)) as privileges
|
|
230
|
+
FROM information_schema.column_privileges cpp
|
|
231
|
+
WHERE table_schema ${sql}
|
|
232
|
+
GROUP BY table_schema, table_name, column_name
|
|
233
|
+
) cp
|
|
234
|
+
ON c.table_schema = cp.table_schema AND c.table_name = cp.table_name AND c.column_name = cp.column_name
|
|
235
|
+
) ccc
|
|
236
|
+
WHERE table_schema ${sql}
|
|
237
|
+
ORDER BY table_oid, ordinal_position
|
|
238
|
+
`, { schemaNames });
|
|
239
|
+
|
|
240
|
+
return { columns };
|
|
241
|
+
});
|
|
242
|
+
if(getTVColumns.error || !getTVColumns.columns){
|
|
243
|
+
throw getTVColumns.error ?? "No columns";
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
const getViewParentTables = await tryCatch(async () => {
|
|
247
|
+
const parent_tables: { oid: number; table_names: string[]; }[] = await t.any(`
|
|
248
|
+
SELECT cl_r.oid, cl_r.relname as view_name, array_agg(DISTINCT cl_d.relname) AS table_names
|
|
249
|
+
FROM pg_rewrite AS r
|
|
250
|
+
JOIN pg_class AS cl_r ON r.ev_class = cl_r.oid
|
|
251
|
+
JOIN pg_depend AS d ON r.oid = d.objid
|
|
252
|
+
JOIN pg_class AS cl_d ON d.refobjid = cl_d.oid
|
|
253
|
+
WHERE cl_d.relkind IN ('r','v')
|
|
254
|
+
AND cl_d.relname <> cl_r.relname
|
|
255
|
+
GROUP BY cl_r.oid, cl_r.relname
|
|
256
|
+
`);
|
|
257
|
+
return { parent_tables }
|
|
258
|
+
});
|
|
259
|
+
const getTablesAndViews = await tryCatch(async () => {
|
|
260
|
+
|
|
261
|
+
const query = `
|
|
262
|
+
SELECT
|
|
263
|
+
jsonb_build_object(
|
|
264
|
+
'insert', TRUE,
|
|
265
|
+
'select', TRUE,
|
|
266
|
+
'update', TRUE,
|
|
267
|
+
'delete', EXISTS (
|
|
268
|
+
SELECT 1
|
|
269
|
+
FROM information_schema.role_table_grants rg
|
|
270
|
+
WHERE rg.table_name = t.table_name
|
|
271
|
+
AND rg.privilege_type = 'DELETE'
|
|
272
|
+
)
|
|
273
|
+
) as privileges
|
|
274
|
+
, t.table_schema as schema
|
|
275
|
+
, t.table_name as name
|
|
276
|
+
, CASE WHEN current_schema() = t.table_schema
|
|
277
|
+
THEN format('%I', t.table_name)
|
|
278
|
+
ELSE format('%I.%I', t.table_schema, t.table_name)
|
|
279
|
+
END as escaped_identifier
|
|
280
|
+
, t.oid
|
|
281
|
+
, t.is_view
|
|
282
|
+
, CASE WHEN is_view THEN pg_get_viewdef(oid, true) END as view_definition
|
|
283
|
+
, obj_description(t.oid::regclass) as comment
|
|
284
|
+
FROM (
|
|
285
|
+
SELECT table_name
|
|
286
|
+
, table_schema, table_type = 'VIEW' as is_view
|
|
287
|
+
, format('%I.%I', table_schema, table_name)::REGCLASS::oid as oid
|
|
288
|
+
FROM information_schema.tables
|
|
289
|
+
WHERE table_schema ${sql}
|
|
290
|
+
) t
|
|
291
|
+
--GROUP BY t.table_schema, t.table_name, t.is_view, t.view_definition, t.oid
|
|
292
|
+
ORDER BY schema, name
|
|
293
|
+
`;
|
|
294
|
+
const tablesAndViews = (await t.any(query, { schemaNames }) as TableSchema[]).map(table => {
|
|
295
|
+
table.columns = clone(getTVColumns.columns).filter(c => c.table_oid === table.oid).map(c => omitKeys(c, ["table_oid"])) ?? [];
|
|
296
|
+
table.parent_tables = getViewParentTables.parent_tables?.find(vr => vr.oid === table.oid)?.table_names ?? [];
|
|
297
|
+
return table;
|
|
298
|
+
});
|
|
299
|
+
return { tablesAndViews };
|
|
300
|
+
});
|
|
301
|
+
if(getTablesAndViews.error || !getTablesAndViews.tablesAndViews){
|
|
302
|
+
throw getTablesAndViews.error ?? "No tablesAndViews";
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
const getMaterialViewsReq = await tryCatch(async () => {
|
|
306
|
+
const materialViews = await getMaterialViews(t, schema);
|
|
307
|
+
return { materialViews }
|
|
308
|
+
});
|
|
309
|
+
if(getMaterialViewsReq.error || !getMaterialViewsReq.materialViews){
|
|
310
|
+
throw getMaterialViewsReq.error ?? "No materialViews";
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
const getHyperTablesReq = await tryCatch(async () => {
|
|
314
|
+
const hyperTables = await getHyperTables(t);
|
|
315
|
+
return { hyperTables };
|
|
316
|
+
});
|
|
317
|
+
if(getHyperTablesReq.error){
|
|
318
|
+
console.error(getHyperTablesReq.error);
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
let result = getTablesAndViews.tablesAndViews.concat(getMaterialViewsReq.materialViews);
|
|
322
|
+
result = await Promise.all(result
|
|
323
|
+
.map(async table => {
|
|
324
|
+
table.name = table.escaped_identifier;
|
|
325
|
+
/** This is used to prevent bug of table schema not sent */
|
|
326
|
+
const allowAllIfNoColumns = !table.columns?.length? true : undefined;
|
|
327
|
+
table.privileges.select = allowAllIfNoColumns ?? table.columns.some(c => c.privileges.SELECT);
|
|
328
|
+
table.privileges.insert = allowAllIfNoColumns ?? table.columns.some(c => c.privileges.INSERT);
|
|
329
|
+
table.privileges.update = allowAllIfNoColumns ?? table.columns.some(c => c.privileges.UPDATE);
|
|
330
|
+
table.columns = table.columns.map(c => {
|
|
331
|
+
const refs = getFkeys.fkeys!.filter(fc => fc.oid === table.oid && fc.cols.includes(c.name));
|
|
332
|
+
if(refs.length) c.references = refs.map(_ref => {
|
|
333
|
+
const ref = { ..._ref };
|
|
334
|
+
//@ts-ignore
|
|
335
|
+
delete ref.oid;
|
|
336
|
+
return ref;
|
|
337
|
+
});
|
|
338
|
+
return c;
|
|
339
|
+
});
|
|
340
|
+
|
|
341
|
+
/** Get view reference cols (based on parent table) */
|
|
342
|
+
let viewFCols: Pick<TableSchemaColumn, "name" | "references">[] = [];
|
|
343
|
+
if(table.is_view){
|
|
344
|
+
try {
|
|
345
|
+
const view_definition = table.view_definition?.endsWith(";")? table.view_definition.slice(0, -1) : table.view_definition;
|
|
346
|
+
const { fields } = await runSQL(`SELECT * FROM \n ( ${view_definition!} \n) t LIMIT 0`, {}, {}, undefined) as SQLResult<undefined>;
|
|
347
|
+
const ftables = result.filter(r => fields.some(f => f.tableID === r.oid));
|
|
348
|
+
ftables.forEach(ft => {
|
|
349
|
+
const fFields = fields.filter(f => f.tableID === ft.oid);
|
|
350
|
+
const pkeys = ft.columns.filter(c => c.is_pkey);
|
|
351
|
+
const fFieldPK = fFields.filter(ff => pkeys.some(p => p.name === ff.columnName));
|
|
352
|
+
const refCols = pkeys.length && fFieldPK.length === pkeys.length? fFieldPK : fFields.filter(ff => !["json", "jsonb", "xml"].includes(ff.udt_name));
|
|
353
|
+
const _fcols: typeof viewFCols = refCols.map(ff => {
|
|
354
|
+
const d: Pick<TableSchemaColumn, "name" | "references"> = {
|
|
355
|
+
name: ff.columnName!,
|
|
356
|
+
references: [{
|
|
357
|
+
ftable: ft.name,
|
|
358
|
+
fcols: [ff.columnName!],
|
|
359
|
+
cols: [ff.name]
|
|
360
|
+
}]
|
|
361
|
+
}
|
|
362
|
+
return d;
|
|
363
|
+
})
|
|
364
|
+
viewFCols = [
|
|
365
|
+
...viewFCols,
|
|
366
|
+
..._fcols
|
|
367
|
+
];
|
|
368
|
+
});
|
|
369
|
+
} catch(err){
|
|
370
|
+
console.error(err);
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
table.columns = table.columns.map(col => {
|
|
375
|
+
if (col.has_default) {
|
|
376
|
+
/** Hide pkey default value */
|
|
377
|
+
col.column_default = (col.udt_name !== "uuid" && !col.is_pkey && !col.column_default.startsWith("nextval(")) ? col.column_default : null;
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
const viewFCol = viewFCols?.find(fc => fc.name === col.name)
|
|
381
|
+
if(viewFCol){
|
|
382
|
+
col.references = viewFCol.references;
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
return col;
|
|
386
|
+
|
|
387
|
+
});
|
|
388
|
+
table.isHyperTable = getHyperTablesReq.hyperTables?.includes(table.name);
|
|
389
|
+
|
|
390
|
+
return table;
|
|
391
|
+
}));
|
|
392
|
+
|
|
393
|
+
const res = {
|
|
394
|
+
result,
|
|
395
|
+
durations: {
|
|
396
|
+
matv: getMaterialViewsReq.duration,
|
|
397
|
+
columns: getTVColumns.duration,
|
|
398
|
+
tablesAndViews: getTablesAndViews.duration,
|
|
399
|
+
fkeys: getFkeys.duration,
|
|
400
|
+
getHyperTbls: getHyperTablesReq.duration,
|
|
401
|
+
viewParentTbls: getViewParentTables.duration,
|
|
402
|
+
}
|
|
403
|
+
};
|
|
404
|
+
|
|
405
|
+
return res;
|
|
406
|
+
});
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
/**
|
|
410
|
+
* Used to check for Timescale Bug
|
|
411
|
+
*/
|
|
412
|
+
const getHyperTables = async (db: DBorTx): Promise<string[] | undefined> => {
|
|
413
|
+
const schema = "_timescaledb_catalog";
|
|
414
|
+
const res = await db.oneOrNone("SELECT EXISTS( \
|
|
415
|
+
SELECT * \
|
|
416
|
+
FROM information_schema.tables \
|
|
417
|
+
WHERE 1 = 1 \
|
|
418
|
+
AND table_schema = ${schema} \
|
|
419
|
+
AND table_name = 'hypertable' \
|
|
420
|
+
);", { schema });
|
|
421
|
+
if (res.exists) {
|
|
422
|
+
const tables: {table_name: string}[] = await db.any("SELECT table_name FROM " + asName(schema) + ".hypertable;");
|
|
423
|
+
return tables.map(t => t.table_name);
|
|
424
|
+
}
|
|
425
|
+
return undefined;
|
|
426
|
+
}
|