pg-data-yaml 0.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pg_data_yaml-0.0.0/LICENSE +29 -0
- pg_data_yaml-0.0.0/PKG-INFO +265 -0
- pg_data_yaml-0.0.0/README.md +220 -0
- pg_data_yaml-0.0.0/pg_data_yaml/__init__.py +1 -0
- pg_data_yaml-0.0.0/pg_data_yaml/analyze_envs.py +75 -0
- pg_data_yaml-0.0.0/pg_data_yaml/extractor.py +46 -0
- pg_data_yaml-0.0.0/pg_data_yaml/formatter.py +49 -0
- pg_data_yaml-0.0.0/pg_data_yaml/main.py +198 -0
- pg_data_yaml-0.0.0/pg_data_yaml/merge_envs.py +128 -0
- pg_data_yaml-0.0.0/pg_data_yaml/paths.py +13 -0
- pg_data_yaml-0.0.0/pg_data_yaml/pg.py +75 -0
- pg_data_yaml-0.0.0/pg_data_yaml/registry.py +158 -0
- pg_data_yaml-0.0.0/pg_data_yaml/str_diff.py +22 -0
- pg_data_yaml-0.0.0/pg_data_yaml/sync_comment.py +60 -0
- pg_data_yaml-0.0.0/pg_data_yaml/synchronizer.py +230 -0
- pg_data_yaml-0.0.0/pg_data_yaml/values.py +92 -0
- pg_data_yaml-0.0.0/pg_data_yaml.egg-info/PKG-INFO +265 -0
- pg_data_yaml-0.0.0/pg_data_yaml.egg-info/SOURCES.txt +28 -0
- pg_data_yaml-0.0.0/pg_data_yaml.egg-info/dependency_links.txt +1 -0
- pg_data_yaml-0.0.0/pg_data_yaml.egg-info/entry_points.txt +2 -0
- pg_data_yaml-0.0.0/pg_data_yaml.egg-info/not-zip-safe +1 -0
- pg_data_yaml-0.0.0/pg_data_yaml.egg-info/requires.txt +10 -0
- pg_data_yaml-0.0.0/pg_data_yaml.egg-info/top_level.txt +2 -0
- pg_data_yaml-0.0.0/setup.cfg +23 -0
- pg_data_yaml-0.0.0/setup.py +64 -0
- pg_data_yaml-0.0.0/tests/integration/__init__.py +0 -0
- pg_data_yaml-0.0.0/tests/integration/conftest.py +30 -0
- pg_data_yaml-0.0.0/tests/integration/helpers.py +231 -0
- pg_data_yaml-0.0.0/tests/integration/test_export_diff_sync.py +274 -0
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
BSD 3-Clause License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2016, comagic
|
|
4
|
+
All rights reserved.
|
|
5
|
+
|
|
6
|
+
Redistribution and use in source and binary forms, with or without
|
|
7
|
+
modification, are permitted provided that the following conditions are met:
|
|
8
|
+
|
|
9
|
+
* Redistributions of source code must retain the above copyright notice, this
|
|
10
|
+
list of conditions and the following disclaimer.
|
|
11
|
+
|
|
12
|
+
* Redistributions in binary form must reproduce the above copyright notice,
|
|
13
|
+
this list of conditions and the following disclaimer in the documentation
|
|
14
|
+
and/or other materials provided with the distribution.
|
|
15
|
+
|
|
16
|
+
* Neither the name of the copyright holder nor the names of its
|
|
17
|
+
contributors may be used to endorse or promote products derived from
|
|
18
|
+
this software without specific prior written permission.
|
|
19
|
+
|
|
20
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
21
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
22
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
23
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
24
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
25
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
26
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
27
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
28
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
29
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
@@ -0,0 +1,265 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pg-data-yaml
|
|
3
|
+
Version: 0.0.0
|
|
4
|
+
Summary: Yaml data converter between DB and Repo
|
|
5
|
+
Home-page: https://github.com/andruche/pg-data-yaml
|
|
6
|
+
Author: Andrey Chernyakov
|
|
7
|
+
License: BSD
|
|
8
|
+
Project-URL: Documentation, https://github.com/andruche/pg-data-yaml/blob/master/README.md
|
|
9
|
+
Project-URL: Bug Tracker, https://github.com/andruche/pg-data-yaml/issues
|
|
10
|
+
Keywords: postgres,postgresql,yaml,reference-data
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Environment :: Console
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Requires-Python: >=3.8
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
License-File: LICENSE
|
|
23
|
+
Requires-Dist: pyyaml
|
|
24
|
+
Requires-Dist: asyncpg<0.31.0,>=0.27.0
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: flake8<6,>=5; extra == "dev"
|
|
27
|
+
Requires-Dist: pytest<9,>=8; extra == "dev"
|
|
28
|
+
Requires-Dist: pytest-cov; extra == "dev"
|
|
29
|
+
Requires-Dist: pytest-flake8; extra == "dev"
|
|
30
|
+
Requires-Dist: pytest-asyncio; extra == "dev"
|
|
31
|
+
Requires-Dist: pytest-sugar; extra == "dev"
|
|
32
|
+
Dynamic: author
|
|
33
|
+
Dynamic: classifier
|
|
34
|
+
Dynamic: description
|
|
35
|
+
Dynamic: description-content-type
|
|
36
|
+
Dynamic: home-page
|
|
37
|
+
Dynamic: keywords
|
|
38
|
+
Dynamic: license
|
|
39
|
+
Dynamic: license-file
|
|
40
|
+
Dynamic: project-url
|
|
41
|
+
Dynamic: provides-extra
|
|
42
|
+
Dynamic: requires-dist
|
|
43
|
+
Dynamic: requires-python
|
|
44
|
+
Dynamic: summary
|
|
45
|
+
|
|
46
|
+
## pg-data-yaml — Yaml interface for reference tables in PostgreSQL
|
|
47
|
+
|
|
48
|
+
Export, diff and sync rows of reference tables in PostgreSQL to YAML files in a repository.
|
|
49
|
+
|
|
50
|
+
## installation
|
|
51
|
+
|
|
52
|
+
```
|
|
53
|
+
pip install pg-data-yaml
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
## which tables are included
|
|
57
|
+
|
|
58
|
+
Table selection is configured with `--comment-label` or `--table-list-predicate`. One of these options is **required** for `export`, `diff` and `sync` (mutually exclusive).
|
|
59
|
+
|
|
60
|
+
### `--comment-label LABEL`
|
|
61
|
+
|
|
62
|
+
Include tables whose comment contains `LABEL`. Any label can be used, for example `global directory` or `env directory`.
|
|
63
|
+
|
|
64
|
+
Optional clause in parentheses after the label is parsed from the comment (see [marking tables](#marking-tables) below).
|
|
65
|
+
|
|
66
|
+
```
|
|
67
|
+
pg_data_yaml export -d mydb --out-dir /tmp/refs --comment-label "global directory"
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
### `--table-list-predicate PREDICATE`
|
|
71
|
+
|
|
72
|
+
Include tables matching a SQL predicate. Table comments are ignored; export always uses `select * from <schema>.<table> order by <primary key columns>`.
|
|
73
|
+
|
|
74
|
+
The predicate is inserted into a query against `pg_catalog.pg_class` and `pg_catalog.pg_namespace`. Use `n.nspname` for schema name and `c.relname` for table name.
|
|
75
|
+
|
|
76
|
+
Example — tables without tenant columns:
|
|
77
|
+
|
|
78
|
+
```
|
|
79
|
+
pg_data_yaml export -d mydb --out-dir /tmp/refs --table-list-predicate "
|
|
80
|
+
not exists (
|
|
81
|
+
select 1
|
|
82
|
+
from information_schema.columns col
|
|
83
|
+
where col.table_schema = n.nspname
|
|
84
|
+
and col.table_name = c.relname
|
|
85
|
+
and col.column_name in ('app_id', 'customer_id', 'customerid')
|
|
86
|
+
)"
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
Tables must have a primary key. Tables without a PK are skipped with a warning.
|
|
90
|
+
|
|
91
|
+
## marking tables
|
|
92
|
+
|
|
93
|
+
When using `--comment-label`, add a comment on the table (via `COMMENT ON TABLE`):
|
|
94
|
+
|
|
95
|
+
```
|
|
96
|
+
synchronized directory
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
With a custom label:
|
|
100
|
+
|
|
101
|
+
```
|
|
102
|
+
global directory
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
Optional clause in parentheses:
|
|
106
|
+
|
|
107
|
+
- if it contains the word `select` — a full export query, for example:
|
|
108
|
+
|
|
109
|
+
```
|
|
110
|
+
synchronized directory(select id, name from my_schema.my_table order by name)
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
- otherwise — a `WHERE` filter inserted into the default template:
|
|
114
|
+
|
|
115
|
+
```
|
|
116
|
+
synchronized directory(not is_deleted)
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
→ `select * from <schema>.<table> where not is_deleted order by <primary key columns>`
|
|
120
|
+
|
|
121
|
+
If parentheses are omitted, export uses:
|
|
122
|
+
|
|
123
|
+
```
|
|
124
|
+
select * from <schema>.<table> order by <primary key columns>
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
Exported file layout: `<out-dir>/<schema>/<table>.yaml`
|
|
128
|
+
|
|
129
|
+
Each file is a YAML list of row mappings. Row and field order match the query result.
|
|
130
|
+
|
|
131
|
+
## usage
|
|
132
|
+
|
|
133
|
+
### export
|
|
134
|
+
|
|
135
|
+
```
|
|
136
|
+
usage: pg_data_yaml export [--help] [-d DBNAME] [-h HOST] [-p PORT] [-U USER] [-W PASSWORD]
|
|
137
|
+
(--comment-label LABEL | --table-list-predicate PREDICATE)
|
|
138
|
+
--out-dir OUT_DIR [--clean]
|
|
139
|
+
|
|
140
|
+
options:
|
|
141
|
+
--help show this help message and exit
|
|
142
|
+
-d DBNAME, --dbname DBNAME
|
|
143
|
+
database name to connect to
|
|
144
|
+
-h HOST, --host HOST database server host or socket directory
|
|
145
|
+
-p PORT, --port PORT database server port
|
|
146
|
+
-U USER, --user USER database user name
|
|
147
|
+
-W PASSWORD, --password PASSWORD
|
|
148
|
+
database user password
|
|
149
|
+
--comment-label LABEL
|
|
150
|
+
include tables whose comment contains LABEL
|
|
151
|
+
--table-list-predicate PREDICATE
|
|
152
|
+
include tables matching SQL PREDICATE (comments ignored)
|
|
153
|
+
--out-dir OUT_DIR directory for exporting files
|
|
154
|
+
--clean clean out_dir if not empty (env variable DATA_DIRECTORY_AUTOCLEAN=true)
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
### diff
|
|
158
|
+
|
|
159
|
+
```
|
|
160
|
+
usage: pg_data_yaml diff [--help] [-d DBNAME] [-h HOST] [-p PORT] [-U USER] [-W PASSWORD]
|
|
161
|
+
(--comment-label LABEL | --table-list-predicate PREDICATE)
|
|
162
|
+
--source SOURCE
|
|
163
|
+
|
|
164
|
+
options:
|
|
165
|
+
--help show this help message and exit
|
|
166
|
+
-d DBNAME, --dbname DBNAME
|
|
167
|
+
-h HOST, --host HOST database server host or socket directory
|
|
168
|
+
-p PORT, --port PORT database server port
|
|
169
|
+
-U USER, --user USER database user name
|
|
170
|
+
-W PASSWORD, --password PASSWORD
|
|
171
|
+
database user password
|
|
172
|
+
--comment-label LABEL
|
|
173
|
+
include tables whose comment contains LABEL
|
|
174
|
+
--table-list-predicate PREDICATE
|
|
175
|
+
include tables matching SQL PREDICATE (comments ignored)
|
|
176
|
+
--source SOURCE directory or yaml file to compare with the database
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
### sync
|
|
180
|
+
|
|
181
|
+
```
|
|
182
|
+
usage: pg_data_yaml sync [--help] [-d DBNAME] [-h HOST] [-p PORT] [-U USER] [-W PASSWORD]
|
|
183
|
+
(--comment-label LABEL | --table-list-predicate PREDICATE)
|
|
184
|
+
--source SOURCE [--dry-run] [--echo-queries] [-y]
|
|
185
|
+
|
|
186
|
+
options:
|
|
187
|
+
--help show this help message and exit
|
|
188
|
+
-d DBNAME, --dbname DBNAME
|
|
189
|
+
-h HOST, --host HOST database server host or socket directory
|
|
190
|
+
-p PORT, --port PORT database server port
|
|
191
|
+
-U USER, --user USER database user name
|
|
192
|
+
-W PASSWORD, --password PASSWORD
|
|
193
|
+
database user password
|
|
194
|
+
--comment-label LABEL
|
|
195
|
+
include tables whose comment contains LABEL
|
|
196
|
+
--table-list-predicate PREDICATE
|
|
197
|
+
include tables matching SQL PREDICATE (comments ignored)
|
|
198
|
+
--source SOURCE directory or yaml file to sync to the database
|
|
199
|
+
--dry-run test run without real changes
|
|
200
|
+
--echo-queries echo commands sent to server
|
|
201
|
+
-y, --yes do not ask confirm
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
### merge-envs
|
|
205
|
+
|
|
206
|
+
Move table files that are identical in all given environment directories into a shared base directory and remove them from the environment directories.
|
|
207
|
+
|
|
208
|
+
```
|
|
209
|
+
usage: pg_data_yaml merge-envs [--help] --source ENV_DIR [--source ENV_DIR ...] --out-dir OUT_DIR [--dry-run]
|
|
210
|
+
|
|
211
|
+
options:
|
|
212
|
+
--help show this help message and exit
|
|
213
|
+
--source ENV_DIR environment directory (repeat for each env)
|
|
214
|
+
--out-dir OUT_DIR base directory for common table files
|
|
215
|
+
--dry-run show actions without changing files
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
Example layout after merge:
|
|
219
|
+
|
|
220
|
+
```
|
|
221
|
+
refs/
|
|
222
|
+
base/public/countries.yaml # identical in all envs
|
|
223
|
+
dev/public/special.yaml # env-specific
|
|
224
|
+
prod/public/special.yaml
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
## examples
|
|
228
|
+
|
|
229
|
+
Comment label for synchronized reference data:
|
|
230
|
+
|
|
231
|
+
```
|
|
232
|
+
$ pg_data_yaml export -d my_database -h 127.0.0.1 -p 5432 -U postgres \
|
|
233
|
+
--out-dir /tmp/refs/ --comment-label "synchronized directory"
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
Comment label for per-environment data:
|
|
237
|
+
|
|
238
|
+
```
|
|
239
|
+
$ pg_data_yaml export -d my_database --out-dir /tmp/refs/base --comment-label "global directory"
|
|
240
|
+
$ pg_data_yaml export -d my_database --out-dir /tmp/refs/dev --comment-label "env directory"
|
|
241
|
+
```
|
|
242
|
+
|
|
243
|
+
Predicate-based selection:
|
|
244
|
+
|
|
245
|
+
```
|
|
246
|
+
$ pg_data_yaml export -d my_database --out-dir /tmp/refs/ --table-list-predicate "
|
|
247
|
+
not exists (
|
|
248
|
+
select 1 from information_schema.columns col
|
|
249
|
+
where col.table_schema = n.nspname
|
|
250
|
+
and col.table_name = c.relname
|
|
251
|
+
and col.column_name in ('app_id', 'customer_id', 'customerid')
|
|
252
|
+
)"
|
|
253
|
+
```
|
|
254
|
+
|
|
255
|
+
Diff and sync use the same table selection options as export:
|
|
256
|
+
|
|
257
|
+
```
|
|
258
|
+
$ pg_data_yaml merge-envs --source /tmp/refs/dev --source /tmp/refs/prod --out-dir /tmp/refs/base
|
|
259
|
+
$ pg_data_yaml diff -d my_database -h 127.0.0.1 -p 5432 -U postgres --source /tmp/refs/ --comment-label "global directory"
|
|
260
|
+
$ pg_data_yaml sync -d my_database -h 127.0.0.1 -p 5432 -U postgres \
|
|
261
|
+
--source /tmp/refs/ --comment-label "synchronized directory"
|
|
262
|
+
$ pg_data_yaml diff -d my_database -h 127.0.0.1 -p 5432 -U postgres --source /tmp/refs/public/countries.yaml
|
|
263
|
+
```
|
|
264
|
+
|
|
265
|
+
When syncing a directory, a missing yaml file for an included table is treated as an empty table (rows are deleted). When syncing a single file, only that table is compared and updated.
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
## pg-data-yaml — Yaml interface for reference tables in PostgreSQL
|
|
2
|
+
|
|
3
|
+
Export, diff and sync rows of reference tables in PostgreSQL to YAML files in a repository.
|
|
4
|
+
|
|
5
|
+
## installation
|
|
6
|
+
|
|
7
|
+
```
|
|
8
|
+
pip install pg-data-yaml
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## which tables are included
|
|
12
|
+
|
|
13
|
+
Table selection is configured with `--comment-label` or `--table-list-predicate`. One of these options is **required** for `export`, `diff` and `sync` (mutually exclusive).
|
|
14
|
+
|
|
15
|
+
### `--comment-label LABEL`
|
|
16
|
+
|
|
17
|
+
Include tables whose comment contains `LABEL`. Any label can be used, for example `global directory` or `env directory`.
|
|
18
|
+
|
|
19
|
+
Optional clause in parentheses after the label is parsed from the comment (see [marking tables](#marking-tables) below).
|
|
20
|
+
|
|
21
|
+
```
|
|
22
|
+
pg_data_yaml export -d mydb --out-dir /tmp/refs --comment-label "global directory"
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
### `--table-list-predicate PREDICATE`
|
|
26
|
+
|
|
27
|
+
Include tables matching a SQL predicate. Table comments are ignored; export always uses `select * from <schema>.<table> order by <primary key columns>`.
|
|
28
|
+
|
|
29
|
+
The predicate is inserted into a query against `pg_catalog.pg_class` and `pg_catalog.pg_namespace`. Use `n.nspname` for schema name and `c.relname` for table name.
|
|
30
|
+
|
|
31
|
+
Example — tables without tenant columns:
|
|
32
|
+
|
|
33
|
+
```
|
|
34
|
+
pg_data_yaml export -d mydb --out-dir /tmp/refs --table-list-predicate "
|
|
35
|
+
not exists (
|
|
36
|
+
select 1
|
|
37
|
+
from information_schema.columns col
|
|
38
|
+
where col.table_schema = n.nspname
|
|
39
|
+
and col.table_name = c.relname
|
|
40
|
+
and col.column_name in ('app_id', 'customer_id', 'customerid')
|
|
41
|
+
)"
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Tables must have a primary key. Tables without a PK are skipped with a warning.
|
|
45
|
+
|
|
46
|
+
## marking tables
|
|
47
|
+
|
|
48
|
+
When using `--comment-label`, add a comment on the table (via `COMMENT ON TABLE`):
|
|
49
|
+
|
|
50
|
+
```
|
|
51
|
+
synchronized directory
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
With a custom label:
|
|
55
|
+
|
|
56
|
+
```
|
|
57
|
+
global directory
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
Optional clause in parentheses:
|
|
61
|
+
|
|
62
|
+
- if it contains the word `select` — a full export query, for example:
|
|
63
|
+
|
|
64
|
+
```
|
|
65
|
+
synchronized directory(select id, name from my_schema.my_table order by name)
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
- otherwise — a `WHERE` filter inserted into the default template:
|
|
69
|
+
|
|
70
|
+
```
|
|
71
|
+
synchronized directory(not is_deleted)
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
→ `select * from <schema>.<table> where not is_deleted order by <primary key columns>`
|
|
75
|
+
|
|
76
|
+
If parentheses are omitted, export uses:
|
|
77
|
+
|
|
78
|
+
```
|
|
79
|
+
select * from <schema>.<table> order by <primary key columns>
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
Exported file layout: `<out-dir>/<schema>/<table>.yaml`
|
|
83
|
+
|
|
84
|
+
Each file is a YAML list of row mappings. Row and field order match the query result.
|
|
85
|
+
|
|
86
|
+
## usage
|
|
87
|
+
|
|
88
|
+
### export
|
|
89
|
+
|
|
90
|
+
```
|
|
91
|
+
usage: pg_data_yaml export [--help] [-d DBNAME] [-h HOST] [-p PORT] [-U USER] [-W PASSWORD]
|
|
92
|
+
(--comment-label LABEL | --table-list-predicate PREDICATE)
|
|
93
|
+
--out-dir OUT_DIR [--clean]
|
|
94
|
+
|
|
95
|
+
options:
|
|
96
|
+
--help show this help message and exit
|
|
97
|
+
-d DBNAME, --dbname DBNAME
|
|
98
|
+
database name to connect to
|
|
99
|
+
-h HOST, --host HOST database server host or socket directory
|
|
100
|
+
-p PORT, --port PORT database server port
|
|
101
|
+
-U USER, --user USER database user name
|
|
102
|
+
-W PASSWORD, --password PASSWORD
|
|
103
|
+
database user password
|
|
104
|
+
--comment-label LABEL
|
|
105
|
+
include tables whose comment contains LABEL
|
|
106
|
+
--table-list-predicate PREDICATE
|
|
107
|
+
include tables matching SQL PREDICATE (comments ignored)
|
|
108
|
+
--out-dir OUT_DIR directory for exporting files
|
|
109
|
+
--clean clean out_dir if not empty (env variable DATA_DIRECTORY_AUTOCLEAN=true)
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
### diff
|
|
113
|
+
|
|
114
|
+
```
|
|
115
|
+
usage: pg_data_yaml diff [--help] [-d DBNAME] [-h HOST] [-p PORT] [-U USER] [-W PASSWORD]
|
|
116
|
+
(--comment-label LABEL | --table-list-predicate PREDICATE)
|
|
117
|
+
--source SOURCE
|
|
118
|
+
|
|
119
|
+
options:
|
|
120
|
+
--help show this help message and exit
|
|
121
|
+
-d DBNAME, --dbname DBNAME
|
|
122
|
+
-h HOST, --host HOST database server host or socket directory
|
|
123
|
+
-p PORT, --port PORT database server port
|
|
124
|
+
-U USER, --user USER database user name
|
|
125
|
+
-W PASSWORD, --password PASSWORD
|
|
126
|
+
database user password
|
|
127
|
+
--comment-label LABEL
|
|
128
|
+
include tables whose comment contains LABEL
|
|
129
|
+
--table-list-predicate PREDICATE
|
|
130
|
+
include tables matching SQL PREDICATE (comments ignored)
|
|
131
|
+
--source SOURCE directory or yaml file to compare with the database
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
### sync
|
|
135
|
+
|
|
136
|
+
```
|
|
137
|
+
usage: pg_data_yaml sync [--help] [-d DBNAME] [-h HOST] [-p PORT] [-U USER] [-W PASSWORD]
|
|
138
|
+
(--comment-label LABEL | --table-list-predicate PREDICATE)
|
|
139
|
+
--source SOURCE [--dry-run] [--echo-queries] [-y]
|
|
140
|
+
|
|
141
|
+
options:
|
|
142
|
+
--help show this help message and exit
|
|
143
|
+
-d DBNAME, --dbname DBNAME
|
|
144
|
+
-h HOST, --host HOST database server host or socket directory
|
|
145
|
+
-p PORT, --port PORT database server port
|
|
146
|
+
-U USER, --user USER database user name
|
|
147
|
+
-W PASSWORD, --password PASSWORD
|
|
148
|
+
database user password
|
|
149
|
+
--comment-label LABEL
|
|
150
|
+
include tables whose comment contains LABEL
|
|
151
|
+
--table-list-predicate PREDICATE
|
|
152
|
+
include tables matching SQL PREDICATE (comments ignored)
|
|
153
|
+
--source SOURCE directory or yaml file to sync to the database
|
|
154
|
+
--dry-run test run without real changes
|
|
155
|
+
--echo-queries echo commands sent to server
|
|
156
|
+
-y, --yes do not ask confirm
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
### merge-envs
|
|
160
|
+
|
|
161
|
+
Move table files that are identical in all given environment directories into a shared base directory and remove them from the environment directories.
|
|
162
|
+
|
|
163
|
+
```
|
|
164
|
+
usage: pg_data_yaml merge-envs [--help] --source ENV_DIR [--source ENV_DIR ...] --out-dir OUT_DIR [--dry-run]
|
|
165
|
+
|
|
166
|
+
options:
|
|
167
|
+
--help show this help message and exit
|
|
168
|
+
--source ENV_DIR environment directory (repeat for each env)
|
|
169
|
+
--out-dir OUT_DIR base directory for common table files
|
|
170
|
+
--dry-run show actions without changing files
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
Example layout after merge:
|
|
174
|
+
|
|
175
|
+
```
|
|
176
|
+
refs/
|
|
177
|
+
base/public/countries.yaml # identical in all envs
|
|
178
|
+
dev/public/special.yaml # env-specific
|
|
179
|
+
prod/public/special.yaml
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
## examples
|
|
183
|
+
|
|
184
|
+
Comment label for synchronized reference data:
|
|
185
|
+
|
|
186
|
+
```
|
|
187
|
+
$ pg_data_yaml export -d my_database -h 127.0.0.1 -p 5432 -U postgres \
|
|
188
|
+
--out-dir /tmp/refs/ --comment-label "synchronized directory"
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
Comment label for per-environment data:
|
|
192
|
+
|
|
193
|
+
```
|
|
194
|
+
$ pg_data_yaml export -d my_database --out-dir /tmp/refs/base --comment-label "global directory"
|
|
195
|
+
$ pg_data_yaml export -d my_database --out-dir /tmp/refs/dev --comment-label "env directory"
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
Predicate-based selection:
|
|
199
|
+
|
|
200
|
+
```
|
|
201
|
+
$ pg_data_yaml export -d my_database --out-dir /tmp/refs/ --table-list-predicate "
|
|
202
|
+
not exists (
|
|
203
|
+
select 1 from information_schema.columns col
|
|
204
|
+
where col.table_schema = n.nspname
|
|
205
|
+
and col.table_name = c.relname
|
|
206
|
+
and col.column_name in ('app_id', 'customer_id', 'customerid')
|
|
207
|
+
)"
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
Diff and sync use the same table selection options as export:
|
|
211
|
+
|
|
212
|
+
```
|
|
213
|
+
$ pg_data_yaml merge-envs --source /tmp/refs/dev --source /tmp/refs/prod --out-dir /tmp/refs/base
|
|
214
|
+
$ pg_data_yaml diff -d my_database -h 127.0.0.1 -p 5432 -U postgres --source /tmp/refs/ --comment-label "global directory"
|
|
215
|
+
$ pg_data_yaml sync -d my_database -h 127.0.0.1 -p 5432 -U postgres \
|
|
216
|
+
--source /tmp/refs/ --comment-label "synchronized directory"
|
|
217
|
+
$ pg_data_yaml diff -d my_database -h 127.0.0.1 -p 5432 -U postgres --source /tmp/refs/public/countries.yaml
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
When syncing a directory, a missing yaml file for an included table is treated as an empty table (rows are deleted). When syncing a single file, only that table is compared and updated.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = '0.0.0'
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import os
|
|
5
|
+
import sys
|
|
6
|
+
|
|
7
|
+
import yaml
|
|
8
|
+
|
|
9
|
+
from .paths import list_table_yaml_files
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class AnalyzeEnvs:
|
|
13
|
+
def __init__(self, args: argparse.Namespace):
|
|
14
|
+
self.args = args
|
|
15
|
+
self.sources = [os.path.abspath(path) for path in args.source]
|
|
16
|
+
|
|
17
|
+
def run(self) -> None:
|
|
18
|
+
self._validate()
|
|
19
|
+
synchronized_directories = open('/tmp/synchronized_directory.txt').read().split('\n')
|
|
20
|
+
env_files = [list_table_yaml_files(path) for path in self.sources]
|
|
21
|
+
all_paths = set.union(*[set(files) for files in env_files])
|
|
22
|
+
if not all_paths:
|
|
23
|
+
print('Nothing to analyze: no table files across environments')
|
|
24
|
+
return
|
|
25
|
+
|
|
26
|
+
for rel_path in sorted(all_paths):
|
|
27
|
+
lines = {}
|
|
28
|
+
postfix = ''
|
|
29
|
+
for env in self.sources:
|
|
30
|
+
file_path = os.path.join(env, rel_path)
|
|
31
|
+
if os.path.isfile(file_path):
|
|
32
|
+
for line in yaml.safe_load(open(file_path)):
|
|
33
|
+
line = str(line)
|
|
34
|
+
lines[line] = lines.get(line, 0) + 1
|
|
35
|
+
else:
|
|
36
|
+
postfix = '*'
|
|
37
|
+
identical_lines = sum(1 for count in lines.values() if count == len(self.sources))
|
|
38
|
+
table = self.path_to_table_name(rel_path)
|
|
39
|
+
sync_dir = "да" if table in synchronized_directories else ''
|
|
40
|
+
print(f'{table}\t{identical_lines}/{len(lines) - identical_lines}{postfix}\t{sync_dir}')
|
|
41
|
+
|
|
42
|
+
# def run_base(self) -> None:
|
|
43
|
+
# synchronized_directories=open('/tmp/synchronized_directory.txt').read().split('\n')
|
|
44
|
+
# env_files = [list_table_yaml_files(path) for path in self.sources]
|
|
45
|
+
# all_paths = set.union(*[set(files) for files in env_files])
|
|
46
|
+
# if not all_paths:
|
|
47
|
+
# print('Nothing to analyze: no table files across environments')
|
|
48
|
+
# return
|
|
49
|
+
|
|
50
|
+
# for rel_path in sorted(all_paths):
|
|
51
|
+
# env = self.sources[0]
|
|
52
|
+
# file_path = os.path.join(env, rel_path)
|
|
53
|
+
# lines_count = len(yaml.safe_load(open(file_path)))
|
|
54
|
+
# table = self.path_to_table_name(rel_path)
|
|
55
|
+
# sync_dir="да" if table in synchronized_directories else ''
|
|
56
|
+
# print(f'{table}\t{lines_count}/0\t{sync_dir}')
|
|
57
|
+
|
|
58
|
+
def _validate(self) -> None:
|
|
59
|
+
if len(self.sources) < 2:
|
|
60
|
+
print('ERROR: specify at least two --source directories', file=sys.stderr)
|
|
61
|
+
sys.exit(1)
|
|
62
|
+
|
|
63
|
+
seen = set()
|
|
64
|
+
for path in self.sources:
|
|
65
|
+
if path in seen:
|
|
66
|
+
print(f'ERROR: duplicate source directory: {path}', file=sys.stderr)
|
|
67
|
+
sys.exit(1)
|
|
68
|
+
seen.add(path)
|
|
69
|
+
if not os.path.isdir(path):
|
|
70
|
+
print(f'ERROR: source directory not found: {path}', file=sys.stderr)
|
|
71
|
+
sys.exit(1)
|
|
72
|
+
|
|
73
|
+
@staticmethod
|
|
74
|
+
def path_to_table_name(path: str) -> str:
|
|
75
|
+
return '{}.{}'.format(*path[:-5].split('/'))
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
from .formatter import Formatter
|
|
5
|
+
from .pg import Pg
|
|
6
|
+
from .registry import SyncTable, TableRegistry
|
|
7
|
+
from .values import ordered_row
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Extractor:
|
|
11
|
+
ROWS_LIMIT = 50000
|
|
12
|
+
|
|
13
|
+
def __init__(self, args: argparse.Namespace, pg: Pg):
|
|
14
|
+
self.args = args
|
|
15
|
+
self.pg = pg
|
|
16
|
+
self.registry = TableRegistry.from_args(pg, args)
|
|
17
|
+
self.formatter = Formatter()
|
|
18
|
+
|
|
19
|
+
async def export(self) -> None:
|
|
20
|
+
tables = await self.registry.load()
|
|
21
|
+
for sync_table in tables.values():
|
|
22
|
+
rows_count = await self.get_rows_count(sync_table)
|
|
23
|
+
if rows_count == self.ROWS_LIMIT:
|
|
24
|
+
print(f'Table {sync_table.table} has {rows_count} rows, skipping')
|
|
25
|
+
continue
|
|
26
|
+
rows = await self.fetch_rows(sync_table)
|
|
27
|
+
file_name = sync_table.file_path(self.args.out_dir)
|
|
28
|
+
os.makedirs(os.path.dirname(file_name), exist_ok=True)
|
|
29
|
+
self.formatter.dump(rows, file_name)
|
|
30
|
+
|
|
31
|
+
async def fetch_rows(self, sync_table: SyncTable) -> list[dict]:
|
|
32
|
+
rows = await self.pg.fetch(sync_table.select_query())
|
|
33
|
+
return [ordered_row(row) for row in rows]
|
|
34
|
+
|
|
35
|
+
async def get_rows_count(self, sync_table: SyncTable) -> list[dict]:
|
|
36
|
+
query = sync_table.select_query()
|
|
37
|
+
query = f'select count(*) from ({query} limit {self.ROWS_LIMIT}) as t'
|
|
38
|
+
rows = await self.pg.fetch(query)
|
|
39
|
+
return rows[0]['count']
|
|
40
|
+
|
|
41
|
+
async def get_tables_data(self) -> dict[tuple[str, str], list[dict]]:
|
|
42
|
+
tables = await self.registry.load()
|
|
43
|
+
return {
|
|
44
|
+
key: await self.fetch_rows(sync_table)
|
|
45
|
+
for key, sync_table in tables.items()
|
|
46
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
import yaml
|
|
5
|
+
|
|
6
|
+
_INTERVAL_STR_RE = re.compile(r'^-?(\d+ days? )?\d+:\d{2}:\d{2}$')
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class Formatter:
|
|
10
|
+
_representers_registered = False
|
|
11
|
+
|
|
12
|
+
def __init__(self):
|
|
13
|
+
self._register_representers()
|
|
14
|
+
|
|
15
|
+
@classmethod
|
|
16
|
+
def _register_representers(cls):
|
|
17
|
+
if cls._representers_registered:
|
|
18
|
+
return
|
|
19
|
+
|
|
20
|
+
def str_presenter(dumper, data):
|
|
21
|
+
if '\n' in data:
|
|
22
|
+
return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|')
|
|
23
|
+
if _INTERVAL_STR_RE.fullmatch(data):
|
|
24
|
+
return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='')
|
|
25
|
+
return dumper.represent_scalar('tag:yaml.org,2002:str', data)
|
|
26
|
+
|
|
27
|
+
yaml.add_representer(str, str_presenter)
|
|
28
|
+
cls._representers_registered = True
|
|
29
|
+
|
|
30
|
+
@staticmethod
|
|
31
|
+
def dump(data: Any, file_name: str = None):
|
|
32
|
+
Formatter._register_representers()
|
|
33
|
+
if data is None:
|
|
34
|
+
return ''
|
|
35
|
+
file = None
|
|
36
|
+
if file_name:
|
|
37
|
+
file = open(file_name, 'w')
|
|
38
|
+
return yaml.dump(
|
|
39
|
+
data,
|
|
40
|
+
file,
|
|
41
|
+
allow_unicode=True,
|
|
42
|
+
sort_keys=False,
|
|
43
|
+
width=float('inf'),
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
@staticmethod
|
|
47
|
+
def load(file_name: str) -> Any:
|
|
48
|
+
with open(file_name) as file:
|
|
49
|
+
return yaml.safe_load(file)
|