pg-data-yaml 0.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. pg_data_yaml-0.0.0/LICENSE +29 -0
  2. pg_data_yaml-0.0.0/PKG-INFO +265 -0
  3. pg_data_yaml-0.0.0/README.md +220 -0
  4. pg_data_yaml-0.0.0/pg_data_yaml/__init__.py +1 -0
  5. pg_data_yaml-0.0.0/pg_data_yaml/analyze_envs.py +75 -0
  6. pg_data_yaml-0.0.0/pg_data_yaml/extractor.py +46 -0
  7. pg_data_yaml-0.0.0/pg_data_yaml/formatter.py +49 -0
  8. pg_data_yaml-0.0.0/pg_data_yaml/main.py +198 -0
  9. pg_data_yaml-0.0.0/pg_data_yaml/merge_envs.py +128 -0
  10. pg_data_yaml-0.0.0/pg_data_yaml/paths.py +13 -0
  11. pg_data_yaml-0.0.0/pg_data_yaml/pg.py +75 -0
  12. pg_data_yaml-0.0.0/pg_data_yaml/registry.py +158 -0
  13. pg_data_yaml-0.0.0/pg_data_yaml/str_diff.py +22 -0
  14. pg_data_yaml-0.0.0/pg_data_yaml/sync_comment.py +60 -0
  15. pg_data_yaml-0.0.0/pg_data_yaml/synchronizer.py +230 -0
  16. pg_data_yaml-0.0.0/pg_data_yaml/values.py +92 -0
  17. pg_data_yaml-0.0.0/pg_data_yaml.egg-info/PKG-INFO +265 -0
  18. pg_data_yaml-0.0.0/pg_data_yaml.egg-info/SOURCES.txt +28 -0
  19. pg_data_yaml-0.0.0/pg_data_yaml.egg-info/dependency_links.txt +1 -0
  20. pg_data_yaml-0.0.0/pg_data_yaml.egg-info/entry_points.txt +2 -0
  21. pg_data_yaml-0.0.0/pg_data_yaml.egg-info/not-zip-safe +1 -0
  22. pg_data_yaml-0.0.0/pg_data_yaml.egg-info/requires.txt +10 -0
  23. pg_data_yaml-0.0.0/pg_data_yaml.egg-info/top_level.txt +2 -0
  24. pg_data_yaml-0.0.0/setup.cfg +23 -0
  25. pg_data_yaml-0.0.0/setup.py +64 -0
  26. pg_data_yaml-0.0.0/tests/integration/__init__.py +0 -0
  27. pg_data_yaml-0.0.0/tests/integration/conftest.py +30 -0
  28. pg_data_yaml-0.0.0/tests/integration/helpers.py +231 -0
  29. pg_data_yaml-0.0.0/tests/integration/test_export_diff_sync.py +274 -0
@@ -0,0 +1,29 @@
1
+ BSD 3-Clause License
2
+
3
+ Copyright (c) 2016, comagic
4
+ All rights reserved.
5
+
6
+ Redistribution and use in source and binary forms, with or without
7
+ modification, are permitted provided that the following conditions are met:
8
+
9
+ * Redistributions of source code must retain the above copyright notice, this
10
+ list of conditions and the following disclaimer.
11
+
12
+ * Redistributions in binary form must reproduce the above copyright notice,
13
+ this list of conditions and the following disclaimer in the documentation
14
+ and/or other materials provided with the distribution.
15
+
16
+ * Neither the name of the copyright holder nor the names of its
17
+ contributors may be used to endorse or promote products derived from
18
+ this software without specific prior written permission.
19
+
20
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
23
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
24
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
26
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
27
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
28
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
@@ -0,0 +1,265 @@
1
+ Metadata-Version: 2.4
2
+ Name: pg-data-yaml
3
+ Version: 0.0.0
4
+ Summary: Yaml data converter between DB and Repo
5
+ Home-page: https://github.com/andruche/pg-data-yaml
6
+ Author: Andrey Chernyakov
7
+ License: BSD
8
+ Project-URL: Documentation, https://github.com/andruche/pg-data-yaml/blob/master/README.md
9
+ Project-URL: Bug Tracker, https://github.com/andruche/pg-data-yaml/issues
10
+ Keywords: postgres,postgresql,yaml,reference-data
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Environment :: Console
13
+ Classifier: Operating System :: OS Independent
14
+ Classifier: Programming Language :: Python :: 3.8
15
+ Classifier: Programming Language :: Python :: 3.9
16
+ Classifier: Programming Language :: Python :: 3.10
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Programming Language :: Python :: 3.13
20
+ Requires-Python: >=3.8
21
+ Description-Content-Type: text/markdown
22
+ License-File: LICENSE
23
+ Requires-Dist: pyyaml
24
+ Requires-Dist: asyncpg<0.31.0,>=0.27.0
25
+ Provides-Extra: dev
26
+ Requires-Dist: flake8<6,>=5; extra == "dev"
27
+ Requires-Dist: pytest<9,>=8; extra == "dev"
28
+ Requires-Dist: pytest-cov; extra == "dev"
29
+ Requires-Dist: pytest-flake8; extra == "dev"
30
+ Requires-Dist: pytest-asyncio; extra == "dev"
31
+ Requires-Dist: pytest-sugar; extra == "dev"
32
+ Dynamic: author
33
+ Dynamic: classifier
34
+ Dynamic: description
35
+ Dynamic: description-content-type
36
+ Dynamic: home-page
37
+ Dynamic: keywords
38
+ Dynamic: license
39
+ Dynamic: license-file
40
+ Dynamic: project-url
41
+ Dynamic: provides-extra
42
+ Dynamic: requires-dist
43
+ Dynamic: requires-python
44
+ Dynamic: summary
45
+
46
+ ## pg-data-yaml — Yaml interface for reference tables in PostgreSQL
47
+
48
+ Export, diff and sync rows of reference tables in PostgreSQL to YAML files in a repository.
49
+
50
+ ## installation
51
+
52
+ ```
53
+ pip install pg-data-yaml
54
+ ```
55
+
56
+ ## which tables are included
57
+
58
+ Table selection is configured with `--comment-label` or `--table-list-predicate`. One of these options is **required** for `export`, `diff` and `sync` (mutually exclusive).
59
+
60
+ ### `--comment-label LABEL`
61
+
62
+ Include tables whose comment contains `LABEL`. Any label can be used, for example `global directory` or `env directory`.
63
+
64
+ Optional clause in parentheses after the label is parsed from the comment (see [marking tables](#marking-tables) below).
65
+
66
+ ```
67
+ pg_data_yaml export -d mydb --out-dir /tmp/refs --comment-label "global directory"
68
+ ```
69
+
70
+ ### `--table-list-predicate PREDICATE`
71
+
72
+ Include tables matching a SQL predicate. Table comments are ignored; export always uses `select * from <schema>.<table> order by <primary key columns>`.
73
+
74
+ The predicate is inserted into a query against `pg_catalog.pg_class` and `pg_catalog.pg_namespace`. Use `n.nspname` for schema name and `c.relname` for table name.
75
+
76
+ Example — tables without tenant columns:
77
+
78
+ ```
79
+ pg_data_yaml export -d mydb --out-dir /tmp/refs --table-list-predicate "
80
+ not exists (
81
+ select 1
82
+ from information_schema.columns col
83
+ where col.table_schema = n.nspname
84
+ and col.table_name = c.relname
85
+ and col.column_name in ('app_id', 'customer_id', 'customerid')
86
+ )"
87
+ ```
88
+
89
+ Tables must have a primary key. Tables without a PK are skipped with a warning.
90
+
91
+ ## marking tables
92
+
93
+ When using `--comment-label`, add a comment on the table (via `COMMENT ON TABLE`):
94
+
95
+ ```
96
+ synchronized directory
97
+ ```
98
+
99
+ With a custom label:
100
+
101
+ ```
102
+ global directory
103
+ ```
104
+
105
+ Optional clause in parentheses:
106
+
107
+ - if it contains the word `select` — a full export query, for example:
108
+
109
+ ```
110
+ synchronized directory(select id, name from my_schema.my_table order by name)
111
+ ```
112
+
113
+ - otherwise — a `WHERE` filter inserted into the default template:
114
+
115
+ ```
116
+ synchronized directory(not is_deleted)
117
+ ```
118
+
119
+ → `select * from <schema>.<table> where not is_deleted order by <primary key columns>`
120
+
121
+ If parentheses are omitted, export uses:
122
+
123
+ ```
124
+ select * from <schema>.<table> order by <primary key columns>
125
+ ```
126
+
127
+ Exported file layout: `<out-dir>/<schema>/<table>.yaml`
128
+
129
+ Each file is a YAML list of row mappings. Row and field order match the query result.
130
+
131
+ ## usage
132
+
133
+ ### export
134
+
135
+ ```
136
+ usage: pg_data_yaml export [--help] [-d DBNAME] [-h HOST] [-p PORT] [-U USER] [-W PASSWORD]
137
+ (--comment-label LABEL | --table-list-predicate PREDICATE)
138
+ --out-dir OUT_DIR [--clean]
139
+
140
+ options:
141
+ --help show this help message and exit
142
+ -d DBNAME, --dbname DBNAME
143
+ database name to connect to
144
+ -h HOST, --host HOST database server host or socket directory
145
+ -p PORT, --port PORT database server port
146
+ -U USER, --user USER database user name
147
+ -W PASSWORD, --password PASSWORD
148
+ database user password
149
+ --comment-label LABEL
150
+ include tables whose comment contains LABEL
151
+ --table-list-predicate PREDICATE
152
+ include tables matching SQL PREDICATE (comments ignored)
153
+ --out-dir OUT_DIR directory for exporting files
154
+ --clean clean out_dir if not empty (env variable DATA_DIRECTORY_AUTOCLEAN=true)
155
+ ```
156
+
157
+ ### diff
158
+
159
+ ```
160
+ usage: pg_data_yaml diff [--help] [-d DBNAME] [-h HOST] [-p PORT] [-U USER] [-W PASSWORD]
161
+ (--comment-label LABEL | --table-list-predicate PREDICATE)
162
+ --source SOURCE
163
+
164
+ options:
165
+ --help show this help message and exit
166
+ -d DBNAME, --dbname DBNAME
167
+ -h HOST, --host HOST database server host or socket directory
168
+ -p PORT, --port PORT database server port
169
+ -U USER, --user USER database user name
170
+ -W PASSWORD, --password PASSWORD
171
+ database user password
172
+ --comment-label LABEL
173
+ include tables whose comment contains LABEL
174
+ --table-list-predicate PREDICATE
175
+ include tables matching SQL PREDICATE (comments ignored)
176
+ --source SOURCE directory or yaml file to compare with the database
177
+ ```
178
+
179
+ ### sync
180
+
181
+ ```
182
+ usage: pg_data_yaml sync [--help] [-d DBNAME] [-h HOST] [-p PORT] [-U USER] [-W PASSWORD]
183
+ (--comment-label LABEL | --table-list-predicate PREDICATE)
184
+ --source SOURCE [--dry-run] [--echo-queries] [-y]
185
+
186
+ options:
187
+ --help show this help message and exit
188
+ -d DBNAME, --dbname DBNAME
189
+ -h HOST, --host HOST database server host or socket directory
190
+ -p PORT, --port PORT database server port
191
+ -U USER, --user USER database user name
192
+ -W PASSWORD, --password PASSWORD
193
+ database user password
194
+ --comment-label LABEL
195
+ include tables whose comment contains LABEL
196
+ --table-list-predicate PREDICATE
197
+ include tables matching SQL PREDICATE (comments ignored)
198
+ --source SOURCE directory or yaml file to sync to the database
199
+ --dry-run test run without real changes
200
+ --echo-queries echo commands sent to server
201
+ -y, --yes do not ask confirm
202
+ ```
203
+
204
+ ### merge-envs
205
+
206
+ Move table files that are identical in all given environment directories into a shared base directory and remove them from the environment directories.
207
+
208
+ ```
209
+ usage: pg_data_yaml merge-envs [--help] --source ENV_DIR [--source ENV_DIR ...] --out-dir OUT_DIR [--dry-run]
210
+
211
+ options:
212
+ --help show this help message and exit
213
+ --source ENV_DIR environment directory (repeat for each env)
214
+ --out-dir OUT_DIR base directory for common table files
215
+ --dry-run show actions without changing files
216
+ ```
217
+
218
+ Example layout after merge:
219
+
220
+ ```
221
+ refs/
222
+ base/public/countries.yaml # identical in all envs
223
+ dev/public/special.yaml # env-specific
224
+ prod/public/special.yaml
225
+ ```
226
+
227
+ ## examples
228
+
229
+ Comment label for synchronized reference data:
230
+
231
+ ```
232
+ $ pg_data_yaml export -d my_database -h 127.0.0.1 -p 5432 -U postgres \
233
+ --out-dir /tmp/refs/ --comment-label "synchronized directory"
234
+ ```
235
+
236
+ Comment label for per-environment data:
237
+
238
+ ```
239
+ $ pg_data_yaml export -d my_database --out-dir /tmp/refs/base --comment-label "global directory"
240
+ $ pg_data_yaml export -d my_database --out-dir /tmp/refs/dev --comment-label "env directory"
241
+ ```
242
+
243
+ Predicate-based selection:
244
+
245
+ ```
246
+ $ pg_data_yaml export -d my_database --out-dir /tmp/refs/ --table-list-predicate "
247
+ not exists (
248
+ select 1 from information_schema.columns col
249
+ where col.table_schema = n.nspname
250
+ and col.table_name = c.relname
251
+ and col.column_name in ('app_id', 'customer_id', 'customerid')
252
+ )"
253
+ ```
254
+
255
+ Diff and sync use the same table selection options as export:
256
+
257
+ ```
258
+ $ pg_data_yaml merge-envs --source /tmp/refs/dev --source /tmp/refs/prod --out-dir /tmp/refs/base
259
+ $ pg_data_yaml diff -d my_database -h 127.0.0.1 -p 5432 -U postgres --source /tmp/refs/ --comment-label "global directory"
260
+ $ pg_data_yaml sync -d my_database -h 127.0.0.1 -p 5432 -U postgres \
261
+ --source /tmp/refs/ --comment-label "synchronized directory"
262
+ $ pg_data_yaml diff -d my_database -h 127.0.0.1 -p 5432 -U postgres --source /tmp/refs/public/countries.yaml
263
+ ```
264
+
265
+ When syncing a directory, a missing yaml file for an included table is treated as an empty table (rows are deleted). When syncing a single file, only that table is compared and updated.
@@ -0,0 +1,220 @@
1
+ ## pg-data-yaml — Yaml interface for reference tables in PostgreSQL
2
+
3
+ Export, diff and sync rows of reference tables in PostgreSQL to YAML files in a repository.
4
+
5
+ ## installation
6
+
7
+ ```
8
+ pip install pg-data-yaml
9
+ ```
10
+
11
+ ## which tables are included
12
+
13
+ Table selection is configured with `--comment-label` or `--table-list-predicate`. One of these options is **required** for `export`, `diff` and `sync` (mutually exclusive).
14
+
15
+ ### `--comment-label LABEL`
16
+
17
+ Include tables whose comment contains `LABEL`. Any label can be used, for example `global directory` or `env directory`.
18
+
19
+ Optional clause in parentheses after the label is parsed from the comment (see [marking tables](#marking-tables) below).
20
+
21
+ ```
22
+ pg_data_yaml export -d mydb --out-dir /tmp/refs --comment-label "global directory"
23
+ ```
24
+
25
+ ### `--table-list-predicate PREDICATE`
26
+
27
+ Include tables matching a SQL predicate. Table comments are ignored; export always uses `select * from <schema>.<table> order by <primary key columns>`.
28
+
29
+ The predicate is inserted into a query against `pg_catalog.pg_class` and `pg_catalog.pg_namespace`. Use `n.nspname` for schema name and `c.relname` for table name.
30
+
31
+ Example — tables without tenant columns:
32
+
33
+ ```
34
+ pg_data_yaml export -d mydb --out-dir /tmp/refs --table-list-predicate "
35
+ not exists (
36
+ select 1
37
+ from information_schema.columns col
38
+ where col.table_schema = n.nspname
39
+ and col.table_name = c.relname
40
+ and col.column_name in ('app_id', 'customer_id', 'customerid')
41
+ )"
42
+ ```
43
+
44
+ Tables must have a primary key. Tables without a PK are skipped with a warning.
45
+
46
+ ## marking tables
47
+
48
+ When using `--comment-label`, add a comment on the table (via `COMMENT ON TABLE`):
49
+
50
+ ```
51
+ synchronized directory
52
+ ```
53
+
54
+ With a custom label:
55
+
56
+ ```
57
+ global directory
58
+ ```
59
+
60
+ Optional clause in parentheses:
61
+
62
+ - if it contains the word `select` — a full export query, for example:
63
+
64
+ ```
65
+ synchronized directory(select id, name from my_schema.my_table order by name)
66
+ ```
67
+
68
+ - otherwise — a `WHERE` filter inserted into the default template:
69
+
70
+ ```
71
+ synchronized directory(not is_deleted)
72
+ ```
73
+
74
+ → `select * from <schema>.<table> where not is_deleted order by <primary key columns>`
75
+
76
+ If parentheses are omitted, export uses:
77
+
78
+ ```
79
+ select * from <schema>.<table> order by <primary key columns>
80
+ ```
81
+
82
+ Exported file layout: `<out-dir>/<schema>/<table>.yaml`
83
+
84
+ Each file is a YAML list of row mappings. Row and field order match the query result.
85
+
86
+ ## usage
87
+
88
+ ### export
89
+
90
+ ```
91
+ usage: pg_data_yaml export [--help] [-d DBNAME] [-h HOST] [-p PORT] [-U USER] [-W PASSWORD]
92
+ (--comment-label LABEL | --table-list-predicate PREDICATE)
93
+ --out-dir OUT_DIR [--clean]
94
+
95
+ options:
96
+ --help show this help message and exit
97
+ -d DBNAME, --dbname DBNAME
98
+ database name to connect to
99
+ -h HOST, --host HOST database server host or socket directory
100
+ -p PORT, --port PORT database server port
101
+ -U USER, --user USER database user name
102
+ -W PASSWORD, --password PASSWORD
103
+ database user password
104
+ --comment-label LABEL
105
+ include tables whose comment contains LABEL
106
+ --table-list-predicate PREDICATE
107
+ include tables matching SQL PREDICATE (comments ignored)
108
+ --out-dir OUT_DIR directory for exporting files
109
+ --clean clean out_dir if not empty (env variable DATA_DIRECTORY_AUTOCLEAN=true)
110
+ ```
111
+
112
+ ### diff
113
+
114
+ ```
115
+ usage: pg_data_yaml diff [--help] [-d DBNAME] [-h HOST] [-p PORT] [-U USER] [-W PASSWORD]
116
+ (--comment-label LABEL | --table-list-predicate PREDICATE)
117
+ --source SOURCE
118
+
119
+ options:
120
+ --help show this help message and exit
121
+ -d DBNAME, --dbname DBNAME
122
+ -h HOST, --host HOST database server host or socket directory
123
+ -p PORT, --port PORT database server port
124
+ -U USER, --user USER database user name
125
+ -W PASSWORD, --password PASSWORD
126
+ database user password
127
+ --comment-label LABEL
128
+ include tables whose comment contains LABEL
129
+ --table-list-predicate PREDICATE
130
+ include tables matching SQL PREDICATE (comments ignored)
131
+ --source SOURCE directory or yaml file to compare with the database
132
+ ```
133
+
134
+ ### sync
135
+
136
+ ```
137
+ usage: pg_data_yaml sync [--help] [-d DBNAME] [-h HOST] [-p PORT] [-U USER] [-W PASSWORD]
138
+ (--comment-label LABEL | --table-list-predicate PREDICATE)
139
+ --source SOURCE [--dry-run] [--echo-queries] [-y]
140
+
141
+ options:
142
+ --help show this help message and exit
143
+ -d DBNAME, --dbname DBNAME
144
+ -h HOST, --host HOST database server host or socket directory
145
+ -p PORT, --port PORT database server port
146
+ -U USER, --user USER database user name
147
+ -W PASSWORD, --password PASSWORD
148
+ database user password
149
+ --comment-label LABEL
150
+ include tables whose comment contains LABEL
151
+ --table-list-predicate PREDICATE
152
+ include tables matching SQL PREDICATE (comments ignored)
153
+ --source SOURCE directory or yaml file to sync to the database
154
+ --dry-run test run without real changes
155
+ --echo-queries echo commands sent to server
156
+ -y, --yes do not ask confirm
157
+ ```
158
+
159
+ ### merge-envs
160
+
161
+ Move table files that are identical in all given environment directories into a shared base directory and remove them from the environment directories.
162
+
163
+ ```
164
+ usage: pg_data_yaml merge-envs [--help] --source ENV_DIR [--source ENV_DIR ...] --out-dir OUT_DIR [--dry-run]
165
+
166
+ options:
167
+ --help show this help message and exit
168
+ --source ENV_DIR environment directory (repeat for each env)
169
+ --out-dir OUT_DIR base directory for common table files
170
+ --dry-run show actions without changing files
171
+ ```
172
+
173
+ Example layout after merge:
174
+
175
+ ```
176
+ refs/
177
+ base/public/countries.yaml # identical in all envs
178
+ dev/public/special.yaml # env-specific
179
+ prod/public/special.yaml
180
+ ```
181
+
182
+ ## examples
183
+
184
+ Comment label for synchronized reference data:
185
+
186
+ ```
187
+ $ pg_data_yaml export -d my_database -h 127.0.0.1 -p 5432 -U postgres \
188
+ --out-dir /tmp/refs/ --comment-label "synchronized directory"
189
+ ```
190
+
191
+ Comment label for per-environment data:
192
+
193
+ ```
194
+ $ pg_data_yaml export -d my_database --out-dir /tmp/refs/base --comment-label "global directory"
195
+ $ pg_data_yaml export -d my_database --out-dir /tmp/refs/dev --comment-label "env directory"
196
+ ```
197
+
198
+ Predicate-based selection:
199
+
200
+ ```
201
+ $ pg_data_yaml export -d my_database --out-dir /tmp/refs/ --table-list-predicate "
202
+ not exists (
203
+ select 1 from information_schema.columns col
204
+ where col.table_schema = n.nspname
205
+ and col.table_name = c.relname
206
+ and col.column_name in ('app_id', 'customer_id', 'customerid')
207
+ )"
208
+ ```
209
+
210
+ Diff and sync use the same table selection options as export:
211
+
212
+ ```
213
+ $ pg_data_yaml merge-envs --source /tmp/refs/dev --source /tmp/refs/prod --out-dir /tmp/refs/base
214
+ $ pg_data_yaml diff -d my_database -h 127.0.0.1 -p 5432 -U postgres --source /tmp/refs/ --comment-label "global directory"
215
+ $ pg_data_yaml sync -d my_database -h 127.0.0.1 -p 5432 -U postgres \
216
+ --source /tmp/refs/ --comment-label "synchronized directory"
217
+ $ pg_data_yaml diff -d my_database -h 127.0.0.1 -p 5432 -U postgres --source /tmp/refs/public/countries.yaml
218
+ ```
219
+
220
+ When syncing a directory, a missing yaml file for an included table is treated as an empty table (rows are deleted). When syncing a single file, only that table is compared and updated.
@@ -0,0 +1 @@
1
+ __version__ = '0.0.0'
@@ -0,0 +1,75 @@
1
+ from __future__ import annotations
2
+
3
+ import argparse
4
+ import os
5
+ import sys
6
+
7
+ import yaml
8
+
9
+ from .paths import list_table_yaml_files
10
+
11
+
12
+ class AnalyzeEnvs:
13
+ def __init__(self, args: argparse.Namespace):
14
+ self.args = args
15
+ self.sources = [os.path.abspath(path) for path in args.source]
16
+
17
+ def run(self) -> None:
18
+ self._validate()
19
+ synchronized_directories = open('/tmp/synchronized_directory.txt').read().split('\n')
20
+ env_files = [list_table_yaml_files(path) for path in self.sources]
21
+ all_paths = set.union(*[set(files) for files in env_files])
22
+ if not all_paths:
23
+ print('Nothing to analyze: no table files across environments')
24
+ return
25
+
26
+ for rel_path in sorted(all_paths):
27
+ lines = {}
28
+ postfix = ''
29
+ for env in self.sources:
30
+ file_path = os.path.join(env, rel_path)
31
+ if os.path.isfile(file_path):
32
+ for line in yaml.safe_load(open(file_path)):
33
+ line = str(line)
34
+ lines[line] = lines.get(line, 0) + 1
35
+ else:
36
+ postfix = '*'
37
+ identical_lines = sum(1 for count in lines.values() if count == len(self.sources))
38
+ table = self.path_to_table_name(rel_path)
39
+ sync_dir = "да" if table in synchronized_directories else ''
40
+ print(f'{table}\t{identical_lines}/{len(lines) - identical_lines}{postfix}\t{sync_dir}')
41
+
42
+ # def run_base(self) -> None:
43
+ # synchronized_directories=open('/tmp/synchronized_directory.txt').read().split('\n')
44
+ # env_files = [list_table_yaml_files(path) for path in self.sources]
45
+ # all_paths = set.union(*[set(files) for files in env_files])
46
+ # if not all_paths:
47
+ # print('Nothing to analyze: no table files across environments')
48
+ # return
49
+
50
+ # for rel_path in sorted(all_paths):
51
+ # env = self.sources[0]
52
+ # file_path = os.path.join(env, rel_path)
53
+ # lines_count = len(yaml.safe_load(open(file_path)))
54
+ # table = self.path_to_table_name(rel_path)
55
+ # sync_dir="да" if table in synchronized_directories else ''
56
+ # print(f'{table}\t{lines_count}/0\t{sync_dir}')
57
+
58
+ def _validate(self) -> None:
59
+ if len(self.sources) < 2:
60
+ print('ERROR: specify at least two --source directories', file=sys.stderr)
61
+ sys.exit(1)
62
+
63
+ seen = set()
64
+ for path in self.sources:
65
+ if path in seen:
66
+ print(f'ERROR: duplicate source directory: {path}', file=sys.stderr)
67
+ sys.exit(1)
68
+ seen.add(path)
69
+ if not os.path.isdir(path):
70
+ print(f'ERROR: source directory not found: {path}', file=sys.stderr)
71
+ sys.exit(1)
72
+
73
+ @staticmethod
74
+ def path_to_table_name(path: str) -> str:
75
+ return '{}.{}'.format(*path[:-5].split('/'))
@@ -0,0 +1,46 @@
1
+ import argparse
2
+ import os
3
+
4
+ from .formatter import Formatter
5
+ from .pg import Pg
6
+ from .registry import SyncTable, TableRegistry
7
+ from .values import ordered_row
8
+
9
+
10
+ class Extractor:
11
+ ROWS_LIMIT = 50000
12
+
13
+ def __init__(self, args: argparse.Namespace, pg: Pg):
14
+ self.args = args
15
+ self.pg = pg
16
+ self.registry = TableRegistry.from_args(pg, args)
17
+ self.formatter = Formatter()
18
+
19
+ async def export(self) -> None:
20
+ tables = await self.registry.load()
21
+ for sync_table in tables.values():
22
+ rows_count = await self.get_rows_count(sync_table)
23
+ if rows_count == self.ROWS_LIMIT:
24
+ print(f'Table {sync_table.table} has {rows_count} rows, skipping')
25
+ continue
26
+ rows = await self.fetch_rows(sync_table)
27
+ file_name = sync_table.file_path(self.args.out_dir)
28
+ os.makedirs(os.path.dirname(file_name), exist_ok=True)
29
+ self.formatter.dump(rows, file_name)
30
+
31
+ async def fetch_rows(self, sync_table: SyncTable) -> list[dict]:
32
+ rows = await self.pg.fetch(sync_table.select_query())
33
+ return [ordered_row(row) for row in rows]
34
+
35
+ async def get_rows_count(self, sync_table: SyncTable) -> list[dict]:
36
+ query = sync_table.select_query()
37
+ query = f'select count(*) from ({query} limit {self.ROWS_LIMIT}) as t'
38
+ rows = await self.pg.fetch(query)
39
+ return rows[0]['count']
40
+
41
+ async def get_tables_data(self) -> dict[tuple[str, str], list[dict]]:
42
+ tables = await self.registry.load()
43
+ return {
44
+ key: await self.fetch_rows(sync_table)
45
+ for key, sync_table in tables.items()
46
+ }
@@ -0,0 +1,49 @@
1
+ import re
2
+ from typing import Any
3
+
4
+ import yaml
5
+
6
+ _INTERVAL_STR_RE = re.compile(r'^-?(\d+ days? )?\d+:\d{2}:\d{2}$')
7
+
8
+
9
+ class Formatter:
10
+ _representers_registered = False
11
+
12
+ def __init__(self):
13
+ self._register_representers()
14
+
15
+ @classmethod
16
+ def _register_representers(cls):
17
+ if cls._representers_registered:
18
+ return
19
+
20
+ def str_presenter(dumper, data):
21
+ if '\n' in data:
22
+ return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|')
23
+ if _INTERVAL_STR_RE.fullmatch(data):
24
+ return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='')
25
+ return dumper.represent_scalar('tag:yaml.org,2002:str', data)
26
+
27
+ yaml.add_representer(str, str_presenter)
28
+ cls._representers_registered = True
29
+
30
+ @staticmethod
31
+ def dump(data: Any, file_name: str = None):
32
+ Formatter._register_representers()
33
+ if data is None:
34
+ return ''
35
+ file = None
36
+ if file_name:
37
+ file = open(file_name, 'w')
38
+ return yaml.dump(
39
+ data,
40
+ file,
41
+ allow_unicode=True,
42
+ sort_keys=False,
43
+ width=float('inf'),
44
+ )
45
+
46
+ @staticmethod
47
+ def load(file_name: str) -> Any:
48
+ with open(file_name) as file:
49
+ return yaml.safe_load(file)