execsql2 1.130.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- execsql/__init__.py +3 -0
- execsql/__main__.py +5 -0
- execsql/execsql.py +16627 -0
- execsql2-1.130.1.data/data/execsql2_extras/READ_ME.rst +129 -0
- execsql2-1.130.1.data/data/execsql2_extras/config_settings.sqlite +0 -0
- execsql2-1.130.1.data/data/execsql2_extras/example_config_prompt.sql +159 -0
- execsql2-1.130.1.data/data/execsql2_extras/execsql.conf +289 -0
- execsql2-1.130.1.data/data/execsql2_extras/make_config_db.sql +250 -0
- execsql2-1.130.1.data/data/execsql2_extras/md_compare.sql +630 -0
- execsql2-1.130.1.data/data/execsql2_extras/md_glossary.sql +327 -0
- execsql2-1.130.1.data/data/execsql2_extras/md_upsert.sql +2981 -0
- execsql2-1.130.1.data/data/execsql2_extras/pg_compare.sql +629 -0
- execsql2-1.130.1.data/data/execsql2_extras/pg_glossary.sql +291 -0
- execsql2-1.130.1.data/data/execsql2_extras/pg_upsert.sql +2792 -0
- execsql2-1.130.1.data/data/execsql2_extras/script_template.sql +300 -0
- execsql2-1.130.1.data/data/execsql2_extras/ss_compare.sql +639 -0
- execsql2-1.130.1.data/data/execsql2_extras/ss_glossary.sql +405 -0
- execsql2-1.130.1.data/data/execsql2_extras/ss_upsert.sql +2917 -0
- execsql2-1.130.1.dist-info/METADATA +418 -0
- execsql2-1.130.1.dist-info/RECORD +24 -0
- execsql2-1.130.1.dist-info/WHEEL +4 -0
- execsql2-1.130.1.dist-info/entry_points.txt +2 -0
- execsql2-1.130.1.dist-info/licenses/LICENSE.txt +12 -0
- execsql2-1.130.1.dist-info/licenses/NOTICE +10 -0
|
@@ -0,0 +1,629 @@
|
|
|
1
|
+
-- pg_compare.sql
|
|
2
|
+
--
|
|
3
|
+
-- PURPOSE
|
|
4
|
+
-- Generate Postgres SQL to compare the attributes (non-PK columns) in
|
|
5
|
+
-- a staging table to the equivalently-named base table.
|
|
6
|
+
--
|
|
7
|
+
-- NOTES
|
|
8
|
+
-- 1. The scripts contained in this file that are intended to be called
|
|
9
|
+
-- directly by the user are:
|
|
10
|
+
-- COMPARE_COMMON : Produces SQL that represents comparison
|
|
11
|
+
-- results as categories of "Same", "Different",
|
|
12
|
+
-- "Old", and "New".
|
|
13
|
+
-- COMPARE_COMMON_VALS : Produces SQL that shows both base and
|
|
14
|
+
-- staging values in adjacent columns.
|
|
15
|
+
-- COMPARE_CHANGES : Produces SQL that shows all primary keys in
|
|
16
|
+
-- the staging table plus a column named "changes"
|
|
17
|
+
-- that identifiers whether that row in the
|
|
18
|
+
-- staging table is new, is a changed version of
|
|
19
|
+
-- an existing row, or is identical to an
|
|
20
|
+
-- existing row.
|
|
21
|
+
-- COMPARE_CHANGED : Produces SQL that shows all primary keys in
|
|
22
|
+
-- the base table plus a column named "changed"
|
|
23
|
+
-- that identifiers whether that row exists in
|
|
24
|
+
-- the staging table, has different data in the
|
|
25
|
+
-- staging table, or has identical data in the
|
|
26
|
+
-- staging table.
|
|
27
|
+
-- 2. These scripts query the information schema to obtain
|
|
28
|
+
-- the information needed to generate SQL.
|
|
29
|
+
-- 3. Temporary tables and views created by these scripts all
|
|
30
|
+
-- begin with "cmp_".
|
|
31
|
+
--
|
|
32
|
+
-- AUTHOR
|
|
33
|
+
-- Dreas Nielsen (RDN)
|
|
34
|
+
--
|
|
35
|
+
-- COPYRIGHT AND LICENSE
|
|
36
|
+
-- Copyright (c) 2020 R.Dreas Nielsen
|
|
37
|
+
-- This program is free software: you can redistribute it and/or modify
|
|
38
|
+
-- it under the terms of the GNU General Public License as published by
|
|
39
|
+
-- the Free Software Foundation, either version 3 of the License, or
|
|
40
|
+
-- (at your option) any later version.
|
|
41
|
+
-- This program is distributed in the hope that it will be useful,
|
|
42
|
+
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
43
|
+
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
44
|
+
-- GNU General Public License for more details.
|
|
45
|
+
-- The GNU General Public License is available at <http://www.gnu.org/licenses/>
|
|
46
|
+
--
|
|
47
|
+
-- VERSION
|
|
48
|
+
-- 1.1.1
|
|
49
|
+
--
|
|
50
|
+
-- HISTORY
|
|
51
|
+
-- Date Remarks
|
|
52
|
+
-- ---------- -----------------------------------------------------
|
|
53
|
+
-- 2020-02-26 Created. v. 1.0.0. RDN.
|
|
54
|
+
-- 2020-02-27 Added COMPARE_CHANGES and COMPARE_CHANGED. v. 1.1.0. RDN.
|
|
55
|
+
-- 2020-02-29 Excluded PK columns from output of COMPARE_COMMON_VALS.
|
|
56
|
+
-- v. 1.1.1. RDN.
|
|
57
|
+
-- ==================================================================
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
-- ################################################################
|
|
61
|
+
-- Script COMPARE_COMMON
|
|
62
|
+
-- ===============================================================
|
|
63
|
+
--
|
|
64
|
+
-- Generate SQL to compare the attributes of base and staging
|
|
65
|
+
-- tables for only those primary keys in common. This uses an
|
|
66
|
+
-- inner join between the tables, and does not include any rows
|
|
67
|
+
-- with a primary key that is in one table and not the other.
|
|
68
|
+
--
|
|
69
|
+
-- The output includes a column for every column in the base
|
|
70
|
+
-- table (except those explicitly excluded). Primary key
|
|
71
|
+
-- columns are populated with primary key values. Attribute
|
|
72
|
+
-- columns are populated with one of the following tags:
|
|
73
|
+
-- Same : The value in base and staging tables
|
|
74
|
+
-- are the same.
|
|
75
|
+
-- Different : The values in base and staging tables
|
|
76
|
+
-- are different.
|
|
77
|
+
-- Old : There is a value in the base table but
|
|
78
|
+
-- no value (i.e., null) in the staging table.
|
|
79
|
+
-- New : There is a value in the staging table
|
|
80
|
+
-- but no value (i.e., null) in the base table.
|
|
81
|
+
--
|
|
82
|
+
-- Required input arguments:
|
|
83
|
+
-- base_schema : The name of the base table schema.
|
|
84
|
+
-- staging : The name of the staging schema.
|
|
85
|
+
-- table : The name of the table to compare.
|
|
86
|
+
-- The table must have the same name
|
|
87
|
+
-- in base and staging schemas.
|
|
88
|
+
--
|
|
89
|
+
-- Optional input arguments:
|
|
90
|
+
-- include_cols : Contains a comma-separated list of single-quoted
|
|
91
|
+
-- column names in the base table that are the
|
|
92
|
+
-- *only* attribute columns to be compared.
|
|
93
|
+
-- exclude_cols : Contains a comma-separated list of single-quoted
|
|
94
|
+
-- column names in the base table that are not to
|
|
95
|
+
-- be compared. If 'include_cols' is provided,
|
|
96
|
+
-- 'exclude_cols' is ignored.
|
|
97
|
+
--
|
|
98
|
+
-- Required output arguments:
|
|
99
|
+
-- sql_var : The name of the variable into which
|
|
100
|
+
-- the generated SQL will be stored
|
|
101
|
+
--
|
|
102
|
+
-- Notes
|
|
103
|
+
-- 1. The generated SQL is not terminated with a semicolon.
|
|
104
|
+
-- ===============================================================
|
|
105
|
+
-- !x! BEGIN SCRIPT COMPARE_COMMON with parameters (base_schema, staging, table, sql_var)
|
|
106
|
+
|
|
107
|
+
-- Create a table of primary key columns for the table
|
|
108
|
+
drop table if exists cmp_primary_key_columns cascade;
|
|
109
|
+
select k.column_name, k.ordinal_position
|
|
110
|
+
into temporary table cmp_primary_key_columns
|
|
111
|
+
from information_schema.table_constraints as tc
|
|
112
|
+
inner join information_schema.key_column_usage as k
|
|
113
|
+
on tc.constraint_type = 'PRIMARY KEY'
|
|
114
|
+
and tc.constraint_name = k.constraint_name
|
|
115
|
+
and tc.constraint_catalog = k.constraint_catalog
|
|
116
|
+
and tc.constraint_schema = k.constraint_schema
|
|
117
|
+
and tc.table_schema = k.table_schema
|
|
118
|
+
and tc.table_name = k.table_name
|
|
119
|
+
and tc.constraint_name = k.constraint_name
|
|
120
|
+
where
|
|
121
|
+
k.table_name = '!!#table!!'
|
|
122
|
+
and k.table_schema = '!!#base_schema!!'
|
|
123
|
+
order by k.ordinal_position
|
|
124
|
+
;
|
|
125
|
+
|
|
126
|
+
-- !x! if(not hasrows(cmp_primary_key_columns)) { halt message "Table !!#table!! has no primary key columns." }
|
|
127
|
+
|
|
128
|
+
-- Populate a (temporary) table with the names of the attribute columns
|
|
129
|
+
-- that are to be compared.
|
|
130
|
+
-- Include only those columns from the staging table that are also in the base table.
|
|
131
|
+
-- !x! sub_empty ~col_sel
|
|
132
|
+
-- !x! if(sub_defined(#include_cols))
|
|
133
|
+
-- !x! sub ~col_sel and s.column_name in (!!#include_cols!!)
|
|
134
|
+
-- !x! elseif(sub_defined(#exclude_cols))
|
|
135
|
+
-- !x! sub ~col_sel and s.column_name not in (!!#exclude_cols!!)
|
|
136
|
+
-- !x! endif
|
|
137
|
+
drop table if exists cmp_cols cascade;
|
|
138
|
+
select s.column_name
|
|
139
|
+
into temporary table cmp_cols
|
|
140
|
+
from information_schema.columns as s
|
|
141
|
+
inner join information_schema.columns as b on s.column_name=b.column_name
|
|
142
|
+
left join cmp_primary_key_columns as pk on pk.column_name = s.column_name
|
|
143
|
+
where
|
|
144
|
+
s.table_schema = '!!#staging!!'
|
|
145
|
+
and s.table_name = '!!#table!!'
|
|
146
|
+
and b.table_schema = '!!#base_schema!!'
|
|
147
|
+
and b.table_name = '!!#table!!'
|
|
148
|
+
and pk.column_name is null
|
|
149
|
+
!!~col_sel!!
|
|
150
|
+
order by s.ordinal_position;
|
|
151
|
+
|
|
152
|
+
-- Create the SQL to select primary key columns from the base table.
|
|
153
|
+
drop view if exists cmp_pkexpr cascade;
|
|
154
|
+
create temporary view cmp_pkexpr as
|
|
155
|
+
select
|
|
156
|
+
string_agg('b.' || column_name, ', ')
|
|
157
|
+
from
|
|
158
|
+
cmp_primary_key_columns;
|
|
159
|
+
-- !x! subdata ~pkcolexpr cmp_pkexpr
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
-- Create the SQL for each column to compare the base (b) and staging (s) tables.
|
|
163
|
+
alter table cmp_cols add column sql text;
|
|
164
|
+
update cmp_cols
|
|
165
|
+
set sql =
|
|
166
|
+
'case when b.' || column_name || ' is null then case when s.' || column_name || ' is null then ''Same'' else ''New'' end else case when s.' || column_name || ' is null then ''Old'' else case when b.' || column_name || ' = s.' || column_name || ' then ''Same'' else ''Different'' end end end as ' || column_name
|
|
167
|
+
;
|
|
168
|
+
|
|
169
|
+
-- Create the comparison expression for all columns.
|
|
170
|
+
drop view if exists cmp_compexpr cascade;
|
|
171
|
+
create temporary view cmp_compexpr as
|
|
172
|
+
select
|
|
173
|
+
string_agg(sql, ', ')
|
|
174
|
+
from
|
|
175
|
+
cmp_cols;
|
|
176
|
+
-- !x! subdata ~compexpr cmp_compexpr
|
|
177
|
+
|
|
178
|
+
-- Create a join expression for primary key columns of the base (b) and
|
|
179
|
+
-- staging (s) tables.
|
|
180
|
+
drop view if exists cmp_joinexpr cascade;
|
|
181
|
+
create temporary view cmp_joinexpr as
|
|
182
|
+
select
|
|
183
|
+
string_agg('b.' || column_name || ' = s.' || column_name, ' and ')
|
|
184
|
+
from
|
|
185
|
+
cmp_primary_key_columns;
|
|
186
|
+
-- !x! subdata ~joinexpr cmp_joinexpr
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
-- Create and assign the entire SQL.
|
|
190
|
+
-- !x! sub !!#sql_var!! select !!~pkcolexpr!!, !!~compexpr!! from !!#base_schema!!.!!#table!! as b inner join !!#staging!!.!!#table!! as s on !!~joinexpr!!
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
-- !x! END SCRIPT COMPARE_COMMON
|
|
195
|
+
|
|
196
|
+
-- #################### End of COMPARE_COMMON ###################
|
|
197
|
+
-- ################################################################
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
-- ################################################################
|
|
202
|
+
-- Script COMPARE_COMMON_VALS
|
|
203
|
+
-- ===============================================================
|
|
204
|
+
--
|
|
205
|
+
-- Generate SQL to compare the attributes of base and staging
|
|
206
|
+
-- tables for only those primary keys in common. This uses an
|
|
207
|
+
-- inner join between the tables, and does not include any rows
|
|
208
|
+
-- with a primary key that is in one table and not the other.
|
|
209
|
+
--
|
|
210
|
+
-- The output includes two columns for every attribute column in
|
|
211
|
+
-- the base table (except those explicitly excluded), plus a column
|
|
212
|
+
-- for every primary key column. Primary key
|
|
213
|
+
-- columns are populated with primary key values. The two output
|
|
214
|
+
-- columns for each attribute column in the base table have names that
|
|
215
|
+
-- match the base table column name plus a suffix of either "_old",
|
|
216
|
+
-- representing the base table, or "_new", representing the staging
|
|
217
|
+
-- table.
|
|
218
|
+
--
|
|
219
|
+
-- Required input arguments:
|
|
220
|
+
-- base_schema : The name of the base table schema.
|
|
221
|
+
-- staging : The name of the staging schema.
|
|
222
|
+
-- table : The name of the table to compare.
|
|
223
|
+
-- The table must have the same name
|
|
224
|
+
-- in base and staging schemas.
|
|
225
|
+
--
|
|
226
|
+
-- Optional input arguments:
|
|
227
|
+
-- include_cols : Contains a comma-separated list of single-quoted
|
|
228
|
+
-- column names in the base table that are the
|
|
229
|
+
-- *only* attribute columns to be compared.
|
|
230
|
+
-- exclude_cols : Contains a comma-separated list of single-quoted
|
|
231
|
+
-- column names in the base table that are not to
|
|
232
|
+
-- be compared. If 'include_cols' is provided,
|
|
233
|
+
-- 'exclude_cols' is ignored.
|
|
234
|
+
--
|
|
235
|
+
-- Required output arguments:
|
|
236
|
+
-- sql_var : The name of the variable into which
|
|
237
|
+
-- the generated SQL will be stored
|
|
238
|
+
--
|
|
239
|
+
-- Notes
|
|
240
|
+
-- 1. The generated SQL is not terminated with a semicolon.
|
|
241
|
+
-- ===============================================================
|
|
242
|
+
-- !x! BEGIN SCRIPT COMPARE_COMMON_VALS with parameters (base_schema, staging, table, sql_var)
|
|
243
|
+
|
|
244
|
+
-- Create a table of primary key columns for the table
|
|
245
|
+
drop table if exists cmp_primary_key_columns cascade;
|
|
246
|
+
select k.column_name, k.ordinal_position
|
|
247
|
+
into temporary table cmp_primary_key_columns
|
|
248
|
+
from information_schema.table_constraints as tc
|
|
249
|
+
inner join information_schema.key_column_usage as k
|
|
250
|
+
on tc.constraint_type = 'PRIMARY KEY'
|
|
251
|
+
and tc.constraint_name = k.constraint_name
|
|
252
|
+
and tc.constraint_catalog = k.constraint_catalog
|
|
253
|
+
and tc.constraint_schema = k.constraint_schema
|
|
254
|
+
and tc.table_schema = k.table_schema
|
|
255
|
+
and tc.table_name = k.table_name
|
|
256
|
+
and tc.constraint_name = k.constraint_name
|
|
257
|
+
where
|
|
258
|
+
k.table_name = '!!#table!!'
|
|
259
|
+
and k.table_schema = '!!#base_schema!!'
|
|
260
|
+
order by k.ordinal_position
|
|
261
|
+
;
|
|
262
|
+
|
|
263
|
+
-- !x! if(not hasrows(cmp_primary_key_columns)) { halt message "Table !!#table!! has no primary key columns." }
|
|
264
|
+
|
|
265
|
+
-- Populate a (temporary) table with the names of the attribute columns
|
|
266
|
+
-- that are to be compared.
|
|
267
|
+
-- Include only those columns from the staging table that are also in the base table.
|
|
268
|
+
-- !x! sub_empty ~col_sel
|
|
269
|
+
-- !x! if(sub_defined(#include_cols))
|
|
270
|
+
-- !x! sub ~col_sel and s.column_name in (!!#include_cols!!)
|
|
271
|
+
-- !x! elseif(sub_defined(#exclude_cols))
|
|
272
|
+
-- !x! sub ~col_sel and s.column_name not in (!!#exclude_cols!!)
|
|
273
|
+
-- !x! endif
|
|
274
|
+
drop table if exists cmp_cols cascade;
|
|
275
|
+
select s.column_name
|
|
276
|
+
into temporary table cmp_cols
|
|
277
|
+
from information_schema.columns as s
|
|
278
|
+
inner join information_schema.columns as b on s.column_name=b.column_name
|
|
279
|
+
left join cmp_primary_key_columns as pk on pk.column_name = s.column_name
|
|
280
|
+
where
|
|
281
|
+
s.table_schema = '!!#staging!!'
|
|
282
|
+
and s.table_name = '!!#table!!'
|
|
283
|
+
and b.table_schema = '!!#base_schema!!'
|
|
284
|
+
and b.table_name = '!!#table!!'
|
|
285
|
+
and pk.column_name is null
|
|
286
|
+
!!~col_sel!!
|
|
287
|
+
order by s.ordinal_position;
|
|
288
|
+
|
|
289
|
+
-- Create the SQL to select primary key columns from the base table.
|
|
290
|
+
drop view if exists cmp_pkexpr cascade;
|
|
291
|
+
create temporary view cmp_pkexpr as
|
|
292
|
+
select
|
|
293
|
+
string_agg('b.' || column_name, ', ')
|
|
294
|
+
from
|
|
295
|
+
cmp_primary_key_columns;
|
|
296
|
+
-- !x! subdata ~pkcolexpr cmp_pkexpr
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
-- Create the SQL for each column to compare the base (b) and staging (s) tables.
|
|
300
|
+
alter table cmp_cols add column sql text;
|
|
301
|
+
update cmp_cols
|
|
302
|
+
set sql =
|
|
303
|
+
'b.' || column_name || ' as ' || column_name || '_old, s.' || column_name || ' as ' || column_name || '_new'
|
|
304
|
+
;
|
|
305
|
+
|
|
306
|
+
-- Create the comparison expression for all columns.
|
|
307
|
+
drop view if exists cmp_compexpr cascade;
|
|
308
|
+
create temporary view cmp_compexpr as
|
|
309
|
+
select
|
|
310
|
+
string_agg(sql, ', ')
|
|
311
|
+
from
|
|
312
|
+
cmp_cols;
|
|
313
|
+
-- !x! subdata ~compexpr cmp_compexpr
|
|
314
|
+
|
|
315
|
+
-- Create a join expression for primary key columns of the base (b) and
|
|
316
|
+
-- staging (s) tables.
|
|
317
|
+
drop view if exists cmp_joinexpr cascade;
|
|
318
|
+
create temporary view cmp_joinexpr as
|
|
319
|
+
select
|
|
320
|
+
string_agg('b.' || column_name || ' = s.' || column_name, ' and ')
|
|
321
|
+
from
|
|
322
|
+
cmp_primary_key_columns;
|
|
323
|
+
-- !x! subdata ~joinexpr cmp_joinexpr
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
-- Create and assign the entire SQL.
|
|
327
|
+
-- !x! sub !!#sql_var!! select !!~pkcolexpr!!, !!~compexpr!! from !!#base_schema!!.!!#table!! as b inner join !!#staging!!.!!#table!! as s on !!~joinexpr!!
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
-- !x! END SCRIPT COMPARE_COMMON_VALS
|
|
332
|
+
|
|
333
|
+
-- ############### End of COMPARE_COMMON_VALS ###################
|
|
334
|
+
-- ################################################################
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
-- ################################################################
|
|
341
|
+
-- Script COMPARE_CHANGES
|
|
342
|
+
-- ===============================================================
|
|
343
|
+
--
|
|
344
|
+
-- Generate SQL to characterize every row of the staging table
|
|
345
|
+
-- data as either a new row, a row with changed data, or a row
|
|
346
|
+
-- with data that are unchanged relative to the base table.
|
|
347
|
+
--
|
|
348
|
+
-- The output includes a column for every primary key plus a
|
|
349
|
+
-- column named "changes" with one of the following tags:
|
|
350
|
+
-- NewRow : The primary key is in the staging table
|
|
351
|
+
-- but not in the base table.
|
|
352
|
+
-- Changed : At least one of the attribute values is
|
|
353
|
+
-- different in the staging table.
|
|
354
|
+
-- Unchanged : All attribute values are the same in the
|
|
355
|
+
-- staging and base tables.
|
|
356
|
+
--
|
|
357
|
+
-- Required input arguments:
|
|
358
|
+
-- base_schema : The name of the base table schema.
|
|
359
|
+
-- staging : The name of the staging schema.
|
|
360
|
+
-- table : The name of the table to compare.
|
|
361
|
+
-- The table must have the same name
|
|
362
|
+
-- in base and staging schemas.
|
|
363
|
+
--
|
|
364
|
+
-- Optional input arguments:
|
|
365
|
+
-- include_cols : Contains a comma-separated list of single-quoted
|
|
366
|
+
-- column names in the base table that are the
|
|
367
|
+
-- *only* attribute columns to be compared.
|
|
368
|
+
-- exclude_cols : Contains a comma-separated list of single-quoted
|
|
369
|
+
-- column names in the base table that are not to
|
|
370
|
+
-- be compared. If 'include_cols' is provided,
|
|
371
|
+
-- 'exclude_cols' is ignored.
|
|
372
|
+
--
|
|
373
|
+
-- Required output arguments:
|
|
374
|
+
-- sql_var : The name of the variable into which
|
|
375
|
+
-- the generated SQL will be stored
|
|
376
|
+
--
|
|
377
|
+
-- Notes
|
|
378
|
+
-- 1. The generated SQL is not terminated with a semicolon.
|
|
379
|
+
-- ===============================================================
|
|
380
|
+
-- !x! BEGIN SCRIPT COMPARE_CHANGES with parameters (base_schema, staging, table, sql_var)
|
|
381
|
+
|
|
382
|
+
-- Create a table of primary key columns for the table
|
|
383
|
+
drop table if exists cmp_primary_key_columns cascade;
|
|
384
|
+
select k.column_name, k.ordinal_position
|
|
385
|
+
into temporary table cmp_primary_key_columns
|
|
386
|
+
from information_schema.table_constraints as tc
|
|
387
|
+
inner join information_schema.key_column_usage as k
|
|
388
|
+
on tc.constraint_type = 'PRIMARY KEY'
|
|
389
|
+
and tc.constraint_name = k.constraint_name
|
|
390
|
+
and tc.constraint_catalog = k.constraint_catalog
|
|
391
|
+
and tc.constraint_schema = k.constraint_schema
|
|
392
|
+
and tc.table_schema = k.table_schema
|
|
393
|
+
and tc.table_name = k.table_name
|
|
394
|
+
and tc.constraint_name = k.constraint_name
|
|
395
|
+
where
|
|
396
|
+
k.table_name = '!!#table!!'
|
|
397
|
+
and k.table_schema = '!!#base_schema!!'
|
|
398
|
+
order by k.ordinal_position
|
|
399
|
+
;
|
|
400
|
+
|
|
401
|
+
-- !x! if(not hasrows(cmp_primary_key_columns)) { halt message "Table !!#table!! has no primary key columns." }
|
|
402
|
+
|
|
403
|
+
-- Populate a (temporary) table with the names of the attribute columns
|
|
404
|
+
-- that are to be compared.
|
|
405
|
+
-- Include only those columns from the staging table that are also in the base table.
|
|
406
|
+
-- !x! sub_empty ~col_sel
|
|
407
|
+
-- !x! if(sub_defined(#include_cols))
|
|
408
|
+
-- !x! sub ~col_sel and s.column_name in (!!#include_cols!!)
|
|
409
|
+
-- !x! elseif(sub_defined(#exclude_cols))
|
|
410
|
+
-- !x! sub ~col_sel and s.column_name not in (!!#exclude_cols!!)
|
|
411
|
+
-- !x! endif
|
|
412
|
+
drop table if exists cmp_cols cascade;
|
|
413
|
+
select s.column_name
|
|
414
|
+
into temporary table cmp_cols
|
|
415
|
+
from information_schema.columns as s
|
|
416
|
+
inner join information_schema.columns as b on s.column_name=b.column_name
|
|
417
|
+
left join cmp_primary_key_columns as pk on pk.column_name = s.column_name
|
|
418
|
+
where
|
|
419
|
+
s.table_schema = '!!#staging!!'
|
|
420
|
+
and s.table_name = '!!#table!!'
|
|
421
|
+
and b.table_schema = '!!#base_schema!!'
|
|
422
|
+
and b.table_name = '!!#table!!'
|
|
423
|
+
!!~col_sel!!
|
|
424
|
+
order by s.ordinal_position;
|
|
425
|
+
|
|
426
|
+
-- Create the SQL to select primary key columns from the staging table.
|
|
427
|
+
drop view if exists cmp_pkexpr cascade;
|
|
428
|
+
create temporary view cmp_pkexpr as
|
|
429
|
+
select
|
|
430
|
+
string_agg('s.' || column_name, ', ')
|
|
431
|
+
from
|
|
432
|
+
cmp_primary_key_columns;
|
|
433
|
+
-- !x! subdata ~pkcolexpr cmp_pkexpr
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
-- Create the SQL for each column to compare the base (b) and staging (s) tables.
|
|
437
|
+
-- This generates a value of 1 when there are differences and 0 when there are not.
|
|
438
|
+
alter table cmp_cols add column sql text;
|
|
439
|
+
update cmp_cols
|
|
440
|
+
set sql =
|
|
441
|
+
'case when b.' || column_name || ' is null then case when s.' || column_name || ' is null then 0 else 1 end else case when s.' || column_name || ' is null then 1 else case when b.' || column_name || ' = s.' || column_name || ' then 0 else 1 end end end'
|
|
442
|
+
;
|
|
443
|
+
|
|
444
|
+
-- Create the expression to determine whether there is a difference for any column.
|
|
445
|
+
drop view if exists cmp_compexpr cascade;
|
|
446
|
+
create temporary view cmp_compexpr as
|
|
447
|
+
select
|
|
448
|
+
string_agg(sql, ' + ')
|
|
449
|
+
from
|
|
450
|
+
cmp_cols;
|
|
451
|
+
-- !x! subdata ~compexpr cmp_compexpr
|
|
452
|
+
-- !x! sub ~diffexpr case when !!~compexpr!! = 0 then 'Unchanged' else 'Changed' end
|
|
453
|
+
|
|
454
|
+
-- Create a join expression for primary key columns of the base (b) and
|
|
455
|
+
-- staging (s) tables.
|
|
456
|
+
drop view if exists cmp_joinexpr cascade;
|
|
457
|
+
create temporary view cmp_joinexpr as
|
|
458
|
+
select
|
|
459
|
+
string_agg('b.' || column_name || ' = s.' || column_name, ' and ')
|
|
460
|
+
from
|
|
461
|
+
cmp_primary_key_columns;
|
|
462
|
+
-- !x! subdata ~joinexpr cmp_joinexpr
|
|
463
|
+
|
|
464
|
+
-- Get the name of a single primary key column, aliased to the base table,
|
|
465
|
+
-- to test for row existence in the base table.
|
|
466
|
+
-- !x! subdata pkcol1 cmp_primary_key_columns
|
|
467
|
+
|
|
468
|
+
-- Create the SQL to display either the row tag or the comparison tag.
|
|
469
|
+
-- !x! sub ~changeexpr case when b.!!pkcol1!! is null then 'NewRow' else !!~diffexpr!! end as changes
|
|
470
|
+
|
|
471
|
+
-- Create and assign the entire SQL.
|
|
472
|
+
-- !x! sub !!#sql_var!! select !!~pkcolexpr!!, !!~changeexpr!! from !!#staging!!.!!#table!! as s left join !!#base_schema!!.!!#table!! as b on !!~joinexpr!!
|
|
473
|
+
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
-- !x! END SCRIPT COMPARE_CHANGES
|
|
477
|
+
|
|
478
|
+
-- ################### End of COMPARE_CHANGES ###################
|
|
479
|
+
-- ################################################################
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
|
|
486
|
+
-- ################################################################
|
|
487
|
+
-- Script COMPARE_CHANGED
|
|
488
|
+
-- ===============================================================
|
|
489
|
+
--
|
|
490
|
+
-- Generate SQL to characterize every row of the base table
|
|
491
|
+
-- depending on whether there is a matching row in the staging
|
|
492
|
+
-- table, whether a matching row has new data, or whether a
|
|
493
|
+
-- matching row has identical data.
|
|
494
|
+
--
|
|
495
|
+
-- The output includes a column for every primary key plus a
|
|
496
|
+
-- column named "changed" with one of the following tags:
|
|
497
|
+
-- NoNewRow : The primary key is in the base table
|
|
498
|
+
-- but not in the staging table.
|
|
499
|
+
-- Changed : At least one of the attribute values is
|
|
500
|
+
-- different in the staging table.
|
|
501
|
+
-- Unchanged : All attribute values are the same in the
|
|
502
|
+
-- staging and base tables.
|
|
503
|
+
--
|
|
504
|
+
-- Required input arguments:
|
|
505
|
+
-- base_schema : The name of the base table schema.
|
|
506
|
+
-- staging : The name of the staging schema.
|
|
507
|
+
-- table : The name of the table to compare.
|
|
508
|
+
-- The table must have the same name
|
|
509
|
+
-- in base and staging schemas.
|
|
510
|
+
--
|
|
511
|
+
-- Optional input arguments:
|
|
512
|
+
-- include_cols : Contains a comma-separated list of single-quoted
|
|
513
|
+
-- column names in the base table that are the
|
|
514
|
+
-- *only* attribute columns to be compared.
|
|
515
|
+
-- exclude_cols : Contains a comma-separated list of single-quoted
|
|
516
|
+
-- column names in the base table that are not to
|
|
517
|
+
-- be compared. If 'include_cols' is provided,
|
|
518
|
+
-- 'exclude_cols' is ignored.
|
|
519
|
+
--
|
|
520
|
+
-- Required output arguments:
|
|
521
|
+
-- sql_var : The name of the variable into which
|
|
522
|
+
-- the generated SQL will be stored
|
|
523
|
+
--
|
|
524
|
+
-- Notes
|
|
525
|
+
-- 1. The generated SQL is not terminated with a semicolon.
|
|
526
|
+
-- ===============================================================
|
|
527
|
+
-- !x! BEGIN SCRIPT COMPARE_CHANGED with parameters (base_schema, staging, table, sql_var)
|
|
528
|
+
|
|
529
|
+
-- Create a table of primary key columns for the table
|
|
530
|
+
drop table if exists cmp_primary_key_columns cascade;
|
|
531
|
+
select k.column_name, k.ordinal_position
|
|
532
|
+
into temporary table cmp_primary_key_columns
|
|
533
|
+
from information_schema.table_constraints as tc
|
|
534
|
+
inner join information_schema.key_column_usage as k
|
|
535
|
+
on tc.constraint_type = 'PRIMARY KEY'
|
|
536
|
+
and tc.constraint_name = k.constraint_name
|
|
537
|
+
and tc.constraint_catalog = k.constraint_catalog
|
|
538
|
+
and tc.constraint_schema = k.constraint_schema
|
|
539
|
+
and tc.table_schema = k.table_schema
|
|
540
|
+
and tc.table_name = k.table_name
|
|
541
|
+
and tc.constraint_name = k.constraint_name
|
|
542
|
+
where
|
|
543
|
+
k.table_name = '!!#table!!'
|
|
544
|
+
and k.table_schema = '!!#base_schema!!'
|
|
545
|
+
order by k.ordinal_position
|
|
546
|
+
;
|
|
547
|
+
|
|
548
|
+
-- !x! if(not hasrows(cmp_primary_key_columns)) { halt message "Table !!#table!! has no primary key columns." }
|
|
549
|
+
|
|
550
|
+
-- Populate a (temporary) table with the names of the attribute columns
|
|
551
|
+
-- that are to be compared.
|
|
552
|
+
-- Include only those columns from the staging table that are also in the base table.
|
|
553
|
+
-- !x! sub_empty ~col_sel
|
|
554
|
+
-- !x! if(sub_defined(#include_cols))
|
|
555
|
+
-- !x! sub ~col_sel and s.column_name in (!!#include_cols!!)
|
|
556
|
+
-- !x! elseif(sub_defined(#exclude_cols))
|
|
557
|
+
-- !x! sub ~col_sel and s.column_name not in (!!#exclude_cols!!)
|
|
558
|
+
-- !x! endif
|
|
559
|
+
drop table if exists cmp_cols cascade;
|
|
560
|
+
select s.column_name
|
|
561
|
+
into temporary table cmp_cols
|
|
562
|
+
from information_schema.columns as s
|
|
563
|
+
inner join information_schema.columns as b on s.column_name=b.column_name
|
|
564
|
+
left join cmp_primary_key_columns as pk on pk.column_name = s.column_name
|
|
565
|
+
where
|
|
566
|
+
s.table_schema = '!!#staging!!'
|
|
567
|
+
and s.table_name = '!!#table!!'
|
|
568
|
+
and b.table_schema = '!!#base_schema!!'
|
|
569
|
+
and b.table_name = '!!#table!!'
|
|
570
|
+
!!~col_sel!!
|
|
571
|
+
order by s.ordinal_position;
|
|
572
|
+
|
|
573
|
+
-- Create the SQL to select primary key columns from the base table.
|
|
574
|
+
drop view if exists cmp_pkexpr cascade;
|
|
575
|
+
create temporary view cmp_pkexpr as
|
|
576
|
+
select
|
|
577
|
+
string_agg('b.' || column_name, ', ')
|
|
578
|
+
from
|
|
579
|
+
cmp_primary_key_columns;
|
|
580
|
+
-- !x! subdata ~pkcolexpr cmp_pkexpr
|
|
581
|
+
|
|
582
|
+
|
|
583
|
+
-- Create the SQL for each column to compare the base (b) and staging (s) tables.
|
|
584
|
+
-- This generates a value of 1 when there are differences and 0 when there are not.
|
|
585
|
+
alter table cmp_cols add column sql text;
|
|
586
|
+
update cmp_cols
|
|
587
|
+
set sql =
|
|
588
|
+
'case when b.' || column_name || ' is null then case when s.' || column_name || ' is null then 0 else 1 end else case when s.' || column_name || ' is null then 1 else case when b.' || column_name || ' = s.' || column_name || ' then 0 else 1 end end end'
|
|
589
|
+
;
|
|
590
|
+
|
|
591
|
+
-- Create the expression to determine whether there is a difference for any column.
|
|
592
|
+
drop view if exists cmp_compexpr cascade;
|
|
593
|
+
create temporary view cmp_compexpr as
|
|
594
|
+
select
|
|
595
|
+
string_agg(sql, ' + ')
|
|
596
|
+
from
|
|
597
|
+
cmp_cols;
|
|
598
|
+
-- !x! subdata ~compexpr cmp_compexpr
|
|
599
|
+
-- !x! sub ~diffexpr case when !!~compexpr!! = 0 then 'Unchanged' else 'Changed' end
|
|
600
|
+
|
|
601
|
+
-- Create a join expression for primary key columns of the base (b) and
|
|
602
|
+
-- staging (s) tables.
|
|
603
|
+
drop view if exists cmp_joinexpr cascade;
|
|
604
|
+
create temporary view cmp_joinexpr as
|
|
605
|
+
select
|
|
606
|
+
string_agg('b.' || column_name || ' = s.' || column_name, ' and ')
|
|
607
|
+
from
|
|
608
|
+
cmp_primary_key_columns;
|
|
609
|
+
-- !x! subdata ~joinexpr cmp_joinexpr
|
|
610
|
+
|
|
611
|
+
-- Get the name of a single primary key column to test for row existence in the
|
|
612
|
+
-- staging table.
|
|
613
|
+
-- !x! subdata pkcol1 cmp_primary_key_columns
|
|
614
|
+
|
|
615
|
+
-- Create the SQL to display either the row tag or the comparison tag.
|
|
616
|
+
-- !x! sub ~changeexpr case when s.!!pkcol1!! is null then 'NoNewRow' else !!~diffexpr!! end as changed
|
|
617
|
+
|
|
618
|
+
-- Create and assign the entire SQL.
|
|
619
|
+
-- !x! sub !!#sql_var!! select !!~pkcolexpr!!, !!~changeexpr!! from !!#base_schema!!.!!#table!! as b left join !!#staging!!.!!#table!! as s on !!~joinexpr!!
|
|
620
|
+
|
|
621
|
+
|
|
622
|
+
|
|
623
|
+
-- !x! END SCRIPT COMPARE_CHANGED
|
|
624
|
+
|
|
625
|
+
-- ################### End of COMPARE_CHANGED ###################
|
|
626
|
+
-- ################################################################
|
|
627
|
+
|
|
628
|
+
|
|
629
|
+
|