testgres-pg-probackup2 0.8.2__tar.gz → 0.9.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {testgres_pg_probackup2-0.8.2 → testgres_pg_probackup2-0.9.1}/PKG-INFO +3 -2
- testgres_pg_probackup2-0.9.1/pg_probackup2/__init__.py +0 -0
- testgres_pg_probackup2-0.9.1/pg_probackup2/init_helpers.py +465 -0
- testgres_pg_probackup2-0.9.1/pg_probackup2/storage/__init__.py +0 -0
- {testgres_pg_probackup2-0.8.2 → testgres_pg_probackup2-0.9.1}/setup.py +7 -2
- {testgres_pg_probackup2-0.8.2 → testgres_pg_probackup2-0.9.1}/testgres_pg_probackup2.egg-info/PKG-INFO +3 -2
- {testgres_pg_probackup2-0.8.2 → testgres_pg_probackup2-0.9.1}/testgres_pg_probackup2.egg-info/requires.txt +2 -1
- testgres_pg_probackup2-0.8.2/pg_probackup2/__init__.py +0 -1
- testgres_pg_probackup2-0.8.2/pg_probackup2/init_helpers.py +0 -230
- testgres_pg_probackup2-0.8.2/pg_probackup2/storage/__init__.py +0 -1
- {testgres_pg_probackup2-0.8.2 → testgres_pg_probackup2-0.9.1}/README.md +0 -0
- {testgres_pg_probackup2-0.8.2 → testgres_pg_probackup2-0.9.1}/pg_probackup2/app.py +0 -0
- {testgres_pg_probackup2-0.8.2 → testgres_pg_probackup2-0.9.1}/pg_probackup2/gdb.py +0 -0
- {testgres_pg_probackup2-0.8.2 → testgres_pg_probackup2-0.9.1}/pg_probackup2/storage/fs_backup.py +0 -0
- {testgres_pg_probackup2-0.8.2 → testgres_pg_probackup2-0.9.1}/setup.cfg +0 -0
- {testgres_pg_probackup2-0.8.2 → testgres_pg_probackup2-0.9.1}/testgres_pg_probackup2.egg-info/SOURCES.txt +0 -0
- {testgres_pg_probackup2-0.8.2 → testgres_pg_probackup2-0.9.1}/testgres_pg_probackup2.egg-info/dependency_links.txt +0 -0
- {testgres_pg_probackup2-0.8.2 → testgres_pg_probackup2-0.9.1}/testgres_pg_probackup2.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: testgres_pg_probackup2
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.9.1
|
|
4
4
|
Summary: Plugin for testgres that manages pg_probackup2
|
|
5
5
|
Home-page: https://github.com/postgrespro/testgres
|
|
6
6
|
Author: Postgres Professional
|
|
@@ -8,9 +8,10 @@ Author-email: testgres@postgrespro.ru
|
|
|
8
8
|
License: PostgreSQL
|
|
9
9
|
Keywords: pg_probackup,testing,testgres
|
|
10
10
|
Description-Content-Type: text/markdown
|
|
11
|
+
Requires-Dist: lz4
|
|
12
|
+
Requires-Dist: pefile
|
|
11
13
|
Requires-Dist: testgres>=1.14.4
|
|
12
14
|
Requires-Dist: zstd
|
|
13
|
-
Requires-Dist: lz4
|
|
14
15
|
Dynamic: author
|
|
15
16
|
Dynamic: author-email
|
|
16
17
|
Dynamic: description-content-type
|
|
File without changes
|
|
@@ -0,0 +1,465 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from functools import reduce
|
|
3
|
+
import getpass
|
|
4
|
+
import os
|
|
5
|
+
import pefile
|
|
6
|
+
import re
|
|
7
|
+
import shutil
|
|
8
|
+
import subprocess
|
|
9
|
+
import sys
|
|
10
|
+
import traceback
|
|
11
|
+
import testgres
|
|
12
|
+
import typing
|
|
13
|
+
|
|
14
|
+
try:
|
|
15
|
+
import lz4.frame # noqa: F401
|
|
16
|
+
|
|
17
|
+
HAVE_LZ4 = True
|
|
18
|
+
except ImportError as e:
|
|
19
|
+
HAVE_LZ4 = False
|
|
20
|
+
LZ4_error = e
|
|
21
|
+
|
|
22
|
+
try:
|
|
23
|
+
import zstd # noqa: F401
|
|
24
|
+
|
|
25
|
+
HAVE_ZSTD = True
|
|
26
|
+
except ImportError as e:
|
|
27
|
+
HAVE_ZSTD = False
|
|
28
|
+
ZSTD_error = e
|
|
29
|
+
|
|
30
|
+
delete_logs = os.getenv('KEEP_LOGS') not in ['1', 'y', 'Y']
|
|
31
|
+
|
|
32
|
+
try:
|
|
33
|
+
testgres.configure_testgres(
|
|
34
|
+
cache_initdb=False,
|
|
35
|
+
cached_initdb_dir=False,
|
|
36
|
+
node_cleanup_full=delete_logs)
|
|
37
|
+
except Exception as e:
|
|
38
|
+
logging.warning("Can't configure testgres: {0}".format(e))
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class InitData_ServerProps:
|
|
42
|
+
pgpro_edition: typing.Optional[str]
|
|
43
|
+
num_version: int
|
|
44
|
+
|
|
45
|
+
def __init__(
|
|
46
|
+
self,
|
|
47
|
+
pgpro_edition: typing.Optional[str],
|
|
48
|
+
num_version: int,
|
|
49
|
+
):
|
|
50
|
+
assert pgpro_edition is None or type(pgpro_edition) is str
|
|
51
|
+
assert type(num_version) is int
|
|
52
|
+
self.pgpro_edition = pgpro_edition
|
|
53
|
+
self.num_version = num_version
|
|
54
|
+
return
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class InitData_BinaryName:
|
|
58
|
+
pg_config: str
|
|
59
|
+
postgres: str
|
|
60
|
+
initdb: str
|
|
61
|
+
|
|
62
|
+
def __init__(
|
|
63
|
+
self,
|
|
64
|
+
pg_config: str,
|
|
65
|
+
initdb: str,
|
|
66
|
+
postgres: str,
|
|
67
|
+
):
|
|
68
|
+
assert type(pg_config) is str
|
|
69
|
+
assert type(postgres) is str
|
|
70
|
+
assert type(initdb) is str
|
|
71
|
+
|
|
72
|
+
self.pg_config = pg_config
|
|
73
|
+
self.postgres = postgres
|
|
74
|
+
self.initdb = initdb
|
|
75
|
+
return
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class Init(object):
|
|
79
|
+
_server_props: InitData_ServerProps
|
|
80
|
+
|
|
81
|
+
def __init__(self):
|
|
82
|
+
if '-v' in sys.argv or '--verbose' in sys.argv:
|
|
83
|
+
self.verbose = True
|
|
84
|
+
else:
|
|
85
|
+
self.verbose = False
|
|
86
|
+
|
|
87
|
+
os_ops = testgres.LocalOperations()
|
|
88
|
+
os_name = os_ops.get_name()
|
|
89
|
+
|
|
90
|
+
pg_bin = testgres.get_bin_dir(os_ops)
|
|
91
|
+
if pg_bin is None:
|
|
92
|
+
raise testgres.InvalidOperationException(
|
|
93
|
+
"Failed to determine the Postgres binary directory. Specify the path to the directory in PG_BIN or put it into the system PATH.",
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
if not os_ops.path_exists(pg_bin):
|
|
97
|
+
raise testgres.InvalidOperationException(
|
|
98
|
+
"Path [{}] is not found.".format(pg_bin),
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
pg_config_path = os_ops.build_path(
|
|
102
|
+
pg_bin,
|
|
103
|
+
__class__._get_binary_names(os_ops).pg_config,
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
assert type(pg_config_path) is str
|
|
107
|
+
|
|
108
|
+
if os_ops.path_exists(pg_config_path):
|
|
109
|
+
self._server_props = __class__._init__get_server_props__via_pg_config(
|
|
110
|
+
os_ops,
|
|
111
|
+
pg_config_path,
|
|
112
|
+
)
|
|
113
|
+
else:
|
|
114
|
+
self._server_props = __class__._init__get_server_props__via_server(
|
|
115
|
+
os_ops,
|
|
116
|
+
pg_bin,
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
# TODO: Always test with NLS support and remove this flag
|
|
120
|
+
self.is_nls_enabled = True
|
|
121
|
+
|
|
122
|
+
postgres = os_ops.build_path(pg_bin, 'postgres.exe' if os_name == 'nt' else 'postgres')
|
|
123
|
+
|
|
124
|
+
if os_name == 'posix':
|
|
125
|
+
ldd = os_ops.exec_command(
|
|
126
|
+
['ldd', postgres],
|
|
127
|
+
encoding='utf-8')
|
|
128
|
+
self.is_lz4_enabled = 'liblz4.so' in ldd
|
|
129
|
+
elif os_name == 'nt':
|
|
130
|
+
pe = pefile.PE(postgres, fast_load=True)
|
|
131
|
+
pe.parse_data_directories(directories=[
|
|
132
|
+
pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_IMPORT']])
|
|
133
|
+
self.is_lz4_enabled = False
|
|
134
|
+
for entry in pe.DIRECTORY_ENTRY_IMPORT:
|
|
135
|
+
if entry.dll.decode('utf-8') == 'liblz4.dll':
|
|
136
|
+
self.is_lz4_enabled = True
|
|
137
|
+
break
|
|
138
|
+
else:
|
|
139
|
+
# Fall back to trying to determine lz4 support via pg_config
|
|
140
|
+
pg_config_data = testgres.utils.get_pg_config2(
|
|
141
|
+
pg_config_path=pg_config_path,
|
|
142
|
+
os_ops=os_ops,
|
|
143
|
+
)
|
|
144
|
+
self.is_lz4_enabled = '-llz4' in pg_config_data['LIBS']
|
|
145
|
+
|
|
146
|
+
os.environ['LANGUAGE'] = 'en' # set default locale language to en. All messages will use this locale
|
|
147
|
+
test_env = os.environ.copy()
|
|
148
|
+
envs_list = [
|
|
149
|
+
'LANGUAGE',
|
|
150
|
+
'LC_ALL',
|
|
151
|
+
'PGCONNECT_TIMEOUT',
|
|
152
|
+
'PGDATA',
|
|
153
|
+
'PGDATABASE',
|
|
154
|
+
'PGHOSTADDR',
|
|
155
|
+
'PGREQUIRESSL',
|
|
156
|
+
'PGSERVICE',
|
|
157
|
+
'PGSSLMODE',
|
|
158
|
+
'PGUSER',
|
|
159
|
+
'PGPORT',
|
|
160
|
+
'PGHOST'
|
|
161
|
+
]
|
|
162
|
+
|
|
163
|
+
for e in envs_list:
|
|
164
|
+
test_env.pop(e, None)
|
|
165
|
+
|
|
166
|
+
test_env['LC_MESSAGES'] = 'C'
|
|
167
|
+
test_env['LC_TIME'] = 'C'
|
|
168
|
+
self._test_env = test_env
|
|
169
|
+
|
|
170
|
+
# Get the directory from which the script was executed
|
|
171
|
+
self.source_path = os.getcwd()
|
|
172
|
+
tmp_path = test_env.get('PGPROBACKUP_TMP_DIR')
|
|
173
|
+
if tmp_path and os.path.isabs(tmp_path):
|
|
174
|
+
self.tmp_path = tmp_path
|
|
175
|
+
else:
|
|
176
|
+
self.tmp_path = os.path.abspath(
|
|
177
|
+
os.path.join(self.source_path, tmp_path or os.path.join('tests', 'tmp_dirs'))
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
os.makedirs(self.tmp_path, exist_ok=True)
|
|
181
|
+
|
|
182
|
+
self.username = getpass.getuser()
|
|
183
|
+
|
|
184
|
+
self.probackup_path = None
|
|
185
|
+
if 'PGPROBACKUPBIN' in test_env:
|
|
186
|
+
if shutil.which(test_env["PGPROBACKUPBIN"]):
|
|
187
|
+
self.probackup_path = test_env["PGPROBACKUPBIN"]
|
|
188
|
+
else:
|
|
189
|
+
raise Exception(
|
|
190
|
+
'pg_probackup binary not found at PGPROBACKUPBIN={0}'.format(
|
|
191
|
+
test_env["PGPROBACKUPBIN"]))
|
|
192
|
+
|
|
193
|
+
if not self.probackup_path:
|
|
194
|
+
probackup_path_tmp = os_ops.build_path(pg_bin, 'pg_probackup')
|
|
195
|
+
|
|
196
|
+
if os.path.isfile(probackup_path_tmp):
|
|
197
|
+
if not os.access(probackup_path_tmp, os.X_OK):
|
|
198
|
+
logging.warning('{0} is not an executable file'.format(
|
|
199
|
+
probackup_path_tmp))
|
|
200
|
+
else:
|
|
201
|
+
self.probackup_path = probackup_path_tmp
|
|
202
|
+
|
|
203
|
+
if not self.probackup_path:
|
|
204
|
+
probackup_path_tmp = self.source_path
|
|
205
|
+
|
|
206
|
+
if os.path.isfile(probackup_path_tmp):
|
|
207
|
+
if not os.access(probackup_path_tmp, os.X_OK):
|
|
208
|
+
logging.warning('{0} is not an executable file'.format(
|
|
209
|
+
probackup_path_tmp))
|
|
210
|
+
else:
|
|
211
|
+
self.probackup_path = probackup_path_tmp
|
|
212
|
+
|
|
213
|
+
if not self.probackup_path:
|
|
214
|
+
raise Exception('pg_probackup binary is not found')
|
|
215
|
+
|
|
216
|
+
if os_name == 'posix':
|
|
217
|
+
self.EXTERNAL_DIRECTORY_DELIMITER = ':'
|
|
218
|
+
os.environ['PATH'] = os.path.dirname(
|
|
219
|
+
self.probackup_path) + ':' + os.environ['PATH']
|
|
220
|
+
|
|
221
|
+
elif os_name == 'nt':
|
|
222
|
+
self.EXTERNAL_DIRECTORY_DELIMITER = ';'
|
|
223
|
+
os.environ['PATH'] = os.path.dirname(
|
|
224
|
+
self.probackup_path) + ';' + os.environ['PATH']
|
|
225
|
+
|
|
226
|
+
self.probackup_old_path = None
|
|
227
|
+
if 'PGPROBACKUPBIN_OLD' in test_env:
|
|
228
|
+
if (os.path.isfile(test_env['PGPROBACKUPBIN_OLD']) and os.access(test_env['PGPROBACKUPBIN_OLD'], os.X_OK)):
|
|
229
|
+
self.probackup_old_path = test_env['PGPROBACKUPBIN_OLD']
|
|
230
|
+
else:
|
|
231
|
+
if self.verbose:
|
|
232
|
+
print('PGPROBACKUPBIN_OLD is not an executable file')
|
|
233
|
+
|
|
234
|
+
self.probackup_version = None
|
|
235
|
+
self.old_probackup_version = None
|
|
236
|
+
|
|
237
|
+
probackup_version_output = subprocess.check_output(
|
|
238
|
+
[self.probackup_path, "--version"],
|
|
239
|
+
stderr=subprocess.STDOUT,
|
|
240
|
+
).decode('utf-8')
|
|
241
|
+
match = re.search(r"\d+\.\d+\.\d+",
|
|
242
|
+
probackup_version_output)
|
|
243
|
+
self.probackup_version = match.group(0) if match else None
|
|
244
|
+
match = re.search(r"\(compressions: ([^)]*)\)", probackup_version_output)
|
|
245
|
+
compressions = match.group(1) if match else None
|
|
246
|
+
if compressions:
|
|
247
|
+
self.probackup_compressions = {s.strip() for s in compressions.split(',')}
|
|
248
|
+
else:
|
|
249
|
+
self.probackup_compressions = []
|
|
250
|
+
|
|
251
|
+
if self.probackup_old_path:
|
|
252
|
+
old_probackup_version_output = subprocess.check_output(
|
|
253
|
+
[self.probackup_old_path, "--version"],
|
|
254
|
+
stderr=subprocess.STDOUT,
|
|
255
|
+
).decode('utf-8')
|
|
256
|
+
match = re.search(r"\d+\.\d+\.\d+",
|
|
257
|
+
old_probackup_version_output)
|
|
258
|
+
self.old_probackup_version = match.group(0) if match else None
|
|
259
|
+
|
|
260
|
+
self.remote = test_env.get('PGPROBACKUP_SSH_REMOTE', None) == 'ON'
|
|
261
|
+
self.ptrack = test_env.get('PG_PROBACKUP_PTRACK', None) == 'ON' and self._server_props.num_version >= 110000
|
|
262
|
+
self.wal_tree_enabled = test_env.get('PG_PROBACKUP_WAL_TREE_ENABLED', None) == 'ON'
|
|
263
|
+
|
|
264
|
+
self.bckp_source = test_env.get('PG_PROBACKUP_SOURCE', 'pro').lower()
|
|
265
|
+
if self.bckp_source not in ('base', 'direct', 'pro'):
|
|
266
|
+
raise Exception("Wrong PG_PROBACKUP_SOURCE value. Available options: base|direct|pro")
|
|
267
|
+
|
|
268
|
+
self.paranoia = test_env.get('PG_PROBACKUP_PARANOIA', None) == 'ON'
|
|
269
|
+
env_compress = test_env.get('ARCHIVE_COMPRESSION', None)
|
|
270
|
+
if env_compress:
|
|
271
|
+
env_compress = env_compress.lower()
|
|
272
|
+
if env_compress in ('on', 'zlib'):
|
|
273
|
+
self.compress_suffix = '.gz'
|
|
274
|
+
self.archive_compress = 'zlib'
|
|
275
|
+
elif env_compress == 'lz4':
|
|
276
|
+
if not HAVE_LZ4:
|
|
277
|
+
raise LZ4_error
|
|
278
|
+
if 'lz4' not in self.probackup_compressions:
|
|
279
|
+
raise Exception("pg_probackup is not compiled with lz4 support")
|
|
280
|
+
self.compress_suffix = '.lz4'
|
|
281
|
+
self.archive_compress = 'lz4'
|
|
282
|
+
elif env_compress == 'zstd':
|
|
283
|
+
if not HAVE_ZSTD:
|
|
284
|
+
raise ZSTD_error
|
|
285
|
+
if 'zstd' not in self.probackup_compressions:
|
|
286
|
+
raise Exception("pg_probackup is not compiled with zstd support")
|
|
287
|
+
self.compress_suffix = '.zst'
|
|
288
|
+
self.archive_compress = 'zstd'
|
|
289
|
+
else:
|
|
290
|
+
self.compress_suffix = ''
|
|
291
|
+
self.archive_compress = False
|
|
292
|
+
|
|
293
|
+
cfs_compress = test_env.get('PG_PROBACKUP_CFS_COMPRESS', None)
|
|
294
|
+
if cfs_compress:
|
|
295
|
+
self.cfs_compress = cfs_compress.lower()
|
|
296
|
+
else:
|
|
297
|
+
self.cfs_compress = self.archive_compress
|
|
298
|
+
|
|
299
|
+
os.environ["PGAPPNAME"] = "pg_probackup"
|
|
300
|
+
self.delete_logs = delete_logs
|
|
301
|
+
|
|
302
|
+
if self.probackup_version.split('.')[0].isdigit():
|
|
303
|
+
self.major_version = int(self.probackup_version.split('.')[0])
|
|
304
|
+
else:
|
|
305
|
+
raise Exception('Can\'t process pg_probackup version \"{}\": the major version is expected to be a number'.format(self.probackup_version))
|
|
306
|
+
|
|
307
|
+
self.valgrind = test_env.get('PG_PROBACKUP_VALGRIND')
|
|
308
|
+
self.valgrind_sup_path = test_env.get('PG_PROBACKUP_VALGRIND_SUP')
|
|
309
|
+
|
|
310
|
+
def test_env(self):
|
|
311
|
+
return self._test_env.copy()
|
|
312
|
+
|
|
313
|
+
@property
|
|
314
|
+
def is_enterprise(self) -> bool:
|
|
315
|
+
assert type(self._server_props) is InitData_ServerProps
|
|
316
|
+
return self._server_props.pgpro_edition == 'enterprise'
|
|
317
|
+
|
|
318
|
+
@property
|
|
319
|
+
def is_shardman(self) -> bool:
|
|
320
|
+
assert type(self._server_props) is InitData_ServerProps
|
|
321
|
+
return self._server_props.pgpro_edition == 'shardman'
|
|
322
|
+
|
|
323
|
+
@property
|
|
324
|
+
def is_pgpro(self) -> bool:
|
|
325
|
+
assert type(self._server_props) is InitData_ServerProps
|
|
326
|
+
return self._server_props.pgpro_edition is not None
|
|
327
|
+
|
|
328
|
+
@property
|
|
329
|
+
def pg_config_version(self) -> int:
|
|
330
|
+
assert type(self._server_props) is InitData_ServerProps
|
|
331
|
+
return self._server_props.num_version
|
|
332
|
+
|
|
333
|
+
@staticmethod
|
|
334
|
+
def _init__get_server_props__via_pg_config(
|
|
335
|
+
os_ops: testgres.OsOperations,
|
|
336
|
+
pg_config_path: str,
|
|
337
|
+
) -> InitData_ServerProps:
|
|
338
|
+
assert isinstance(os_ops, testgres.OsOperations)
|
|
339
|
+
assert type(pg_config_path) is str
|
|
340
|
+
|
|
341
|
+
pg_config = testgres.utils.get_pg_config2(
|
|
342
|
+
pg_config_path=pg_config_path,
|
|
343
|
+
os_ops=os_ops,
|
|
344
|
+
)
|
|
345
|
+
assert type(pg_config) is dict
|
|
346
|
+
|
|
347
|
+
pgpro_edition = pg_config.get('PGPRO_EDITION')
|
|
348
|
+
assert pgpro_edition is None or type(pgpro_edition) is str
|
|
349
|
+
|
|
350
|
+
version_str = pg_config.get('VERSION', '')
|
|
351
|
+
assert type(version_str) is str
|
|
352
|
+
|
|
353
|
+
if not version_str:
|
|
354
|
+
raise RuntimeError("Field 'VERSION' not found in pg_config output")
|
|
355
|
+
|
|
356
|
+
version_num = testgres.parse_pg_version(version_str)
|
|
357
|
+
parts = [*version_num.split('.'), '0', '0'][:3]
|
|
358
|
+
parts[0] = re.match(r'\d+', parts[0]).group()
|
|
359
|
+
|
|
360
|
+
num_version = reduce(lambda v, x: v * 100 + int(x), parts, 0)
|
|
361
|
+
assert type(num_version) is int
|
|
362
|
+
|
|
363
|
+
return InitData_ServerProps(
|
|
364
|
+
pgpro_edition=pgpro_edition,
|
|
365
|
+
num_version=num_version,
|
|
366
|
+
)
|
|
367
|
+
|
|
368
|
+
@staticmethod
|
|
369
|
+
def _init__get_server_props__via_server(
|
|
370
|
+
os_ops: testgres.OsOperations,
|
|
371
|
+
pg_bin_dir: str,
|
|
372
|
+
) -> InitData_ServerProps:
|
|
373
|
+
assert isinstance(os_ops, testgres.OsOperations)
|
|
374
|
+
assert type(pg_bin_dir) is str
|
|
375
|
+
|
|
376
|
+
tmpdir = os_ops.mkdtemp()
|
|
377
|
+
assert type(tmpdir) is str
|
|
378
|
+
assert os_ops.path_exists(tmpdir)
|
|
379
|
+
assert os_ops.is_abs_path(tmpdir)
|
|
380
|
+
|
|
381
|
+
try:
|
|
382
|
+
initdb = os_ops.build_path(
|
|
383
|
+
pg_bin_dir,
|
|
384
|
+
__class__._get_binary_names(os_ops).initdb,
|
|
385
|
+
)
|
|
386
|
+
|
|
387
|
+
os_ops.exec_command(
|
|
388
|
+
[initdb, "-D", tmpdir],
|
|
389
|
+
encoding='utf-8',
|
|
390
|
+
)
|
|
391
|
+
|
|
392
|
+
postgres = os_ops.build_path(
|
|
393
|
+
pg_bin_dir,
|
|
394
|
+
__class__._get_binary_names(os_ops).postgres,
|
|
395
|
+
)
|
|
396
|
+
|
|
397
|
+
pgpro_edition_out: typing.Optional[str] = None
|
|
398
|
+
try:
|
|
399
|
+
exec_r = os_ops.exec_command(
|
|
400
|
+
[postgres, "-C", "pgpro_edition", "-D", tmpdir],
|
|
401
|
+
encoding='utf-8',
|
|
402
|
+
)
|
|
403
|
+
assert type(exec_r) is str
|
|
404
|
+
pgpro_edition_out = exec_r.strip()
|
|
405
|
+
except testgres.ExecUtilException as e:
|
|
406
|
+
assert e.exit_code != 0
|
|
407
|
+
|
|
408
|
+
logging.debug("Exception ({}): {}".format(
|
|
409
|
+
type(e).__name__,
|
|
410
|
+
e,
|
|
411
|
+
))
|
|
412
|
+
|
|
413
|
+
assert pgpro_edition_out is None or type(pgpro_edition_out) is str
|
|
414
|
+
pgpro_edition = pgpro_edition_out
|
|
415
|
+
|
|
416
|
+
server_version_out = os_ops.exec_command(
|
|
417
|
+
[postgres, "-C", "server_version", "-D", tmpdir],
|
|
418
|
+
encoding='utf-8'
|
|
419
|
+
)
|
|
420
|
+
assert type(server_version_out) is str
|
|
421
|
+
server_version = testgres.parse_pg_version(server_version_out)
|
|
422
|
+
parts = [*server_version.split('.'), '0', '0'][:3]
|
|
423
|
+
parts[0] = re.match(r'\d+', parts[0]).group()
|
|
424
|
+
# Server_version consists of two fields (x.y) so num_version always ends with 00
|
|
425
|
+
pg_config_version = reduce(lambda v, x: v * 100 + int(x), parts, 0)
|
|
426
|
+
finally:
|
|
427
|
+
os_ops.rmdirs(tmpdir)
|
|
428
|
+
|
|
429
|
+
return InitData_ServerProps(
|
|
430
|
+
pgpro_edition=pgpro_edition,
|
|
431
|
+
num_version=pg_config_version,
|
|
432
|
+
)
|
|
433
|
+
|
|
434
|
+
sm_binary_names__win32 = InitData_BinaryName(
|
|
435
|
+
pg_config="pg_config.exe",
|
|
436
|
+
initdb="initdb.exe",
|
|
437
|
+
postgres="postgres.exe",
|
|
438
|
+
)
|
|
439
|
+
|
|
440
|
+
sm_binary_names__linux = InitData_BinaryName(
|
|
441
|
+
pg_config="pg_config",
|
|
442
|
+
initdb="initdb",
|
|
443
|
+
postgres="postgres",
|
|
444
|
+
)
|
|
445
|
+
|
|
446
|
+
@staticmethod
|
|
447
|
+
def _get_binary_names(
|
|
448
|
+
os_ops: testgres.OsOperations,
|
|
449
|
+
) -> InitData_BinaryName:
|
|
450
|
+
assert isinstance(os_ops, testgres.OsOperations)
|
|
451
|
+
|
|
452
|
+
platform_name = os_ops.get_platform()
|
|
453
|
+
if platform_name == "win32":
|
|
454
|
+
return __class__.sm_binary_names__win32
|
|
455
|
+
|
|
456
|
+
return __class__.sm_binary_names__linux
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
try:
|
|
460
|
+
init_params = Init()
|
|
461
|
+
except Exception as e:
|
|
462
|
+
traceback.print_exc(file=sys.stderr)
|
|
463
|
+
logging.error(str(e))
|
|
464
|
+
logging.warning("testgres.plugins.probackup2.init_params is set to None.")
|
|
465
|
+
init_params = None
|
|
File without changes
|
|
@@ -4,7 +4,7 @@ except ImportError:
|
|
|
4
4
|
from distutils.core import setup
|
|
5
5
|
|
|
6
6
|
setup(
|
|
7
|
-
version='0.
|
|
7
|
+
version='0.9.1',
|
|
8
8
|
name='testgres_pg_probackup2',
|
|
9
9
|
packages=['pg_probackup2', 'pg_probackup2.storage'],
|
|
10
10
|
description='Plugin for testgres that manages pg_probackup2',
|
|
@@ -14,5 +14,10 @@ setup(
|
|
|
14
14
|
author='Postgres Professional',
|
|
15
15
|
author_email='testgres@postgrespro.ru',
|
|
16
16
|
keywords=['pg_probackup', 'testing', 'testgres'],
|
|
17
|
-
install_requires=[
|
|
17
|
+
install_requires=[
|
|
18
|
+
'lz4',
|
|
19
|
+
'pefile',
|
|
20
|
+
'testgres>=1.14.4',
|
|
21
|
+
'zstd'
|
|
22
|
+
]
|
|
18
23
|
)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: testgres_pg_probackup2
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.9.1
|
|
4
4
|
Summary: Plugin for testgres that manages pg_probackup2
|
|
5
5
|
Home-page: https://github.com/postgrespro/testgres
|
|
6
6
|
Author: Postgres Professional
|
|
@@ -8,9 +8,10 @@ Author-email: testgres@postgrespro.ru
|
|
|
8
8
|
License: PostgreSQL
|
|
9
9
|
Keywords: pg_probackup,testing,testgres
|
|
10
10
|
Description-Content-Type: text/markdown
|
|
11
|
+
Requires-Dist: lz4
|
|
12
|
+
Requires-Dist: pefile
|
|
11
13
|
Requires-Dist: testgres>=1.14.4
|
|
12
14
|
Requires-Dist: zstd
|
|
13
|
-
Requires-Dist: lz4
|
|
14
15
|
Dynamic: author
|
|
15
16
|
Dynamic: author-email
|
|
16
17
|
Dynamic: description-content-type
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
|
|
@@ -1,230 +0,0 @@
|
|
|
1
|
-
import logging
|
|
2
|
-
from functools import reduce
|
|
3
|
-
import getpass
|
|
4
|
-
import os
|
|
5
|
-
import re
|
|
6
|
-
import shutil
|
|
7
|
-
import subprocess
|
|
8
|
-
import sys
|
|
9
|
-
import testgres
|
|
10
|
-
|
|
11
|
-
try:
|
|
12
|
-
import lz4.frame # noqa: F401
|
|
13
|
-
|
|
14
|
-
HAVE_LZ4 = True
|
|
15
|
-
except ImportError as e:
|
|
16
|
-
HAVE_LZ4 = False
|
|
17
|
-
LZ4_error = e
|
|
18
|
-
|
|
19
|
-
try:
|
|
20
|
-
import zstd # noqa: F401
|
|
21
|
-
|
|
22
|
-
HAVE_ZSTD = True
|
|
23
|
-
except ImportError as e:
|
|
24
|
-
HAVE_ZSTD = False
|
|
25
|
-
ZSTD_error = e
|
|
26
|
-
|
|
27
|
-
delete_logs = os.getenv('KEEP_LOGS') not in ['1', 'y', 'Y']
|
|
28
|
-
|
|
29
|
-
try:
|
|
30
|
-
testgres.configure_testgres(
|
|
31
|
-
cache_initdb=False,
|
|
32
|
-
cached_initdb_dir=False,
|
|
33
|
-
node_cleanup_full=delete_logs)
|
|
34
|
-
except Exception as e:
|
|
35
|
-
logging.warning("Can't configure testgres: {0}".format(e))
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
class Init(object):
|
|
39
|
-
def __init__(self):
|
|
40
|
-
if '-v' in sys.argv or '--verbose' in sys.argv:
|
|
41
|
-
self.verbose = True
|
|
42
|
-
else:
|
|
43
|
-
self.verbose = False
|
|
44
|
-
|
|
45
|
-
self._pg_config = testgres.get_pg_config()
|
|
46
|
-
self.is_enterprise = self._pg_config.get('PGPRO_EDITION', None) == 'enterprise'
|
|
47
|
-
self.is_shardman = self._pg_config.get('PGPRO_EDITION', None) == 'shardman'
|
|
48
|
-
self.is_pgpro = 'PGPRO_EDITION' in self._pg_config
|
|
49
|
-
self.is_nls_enabled = 'enable-nls' in self._pg_config['CONFIGURE']
|
|
50
|
-
self.is_lz4_enabled = '-llz4' in self._pg_config['LIBS']
|
|
51
|
-
version_num = testgres.parse_pg_version(self._pg_config['VERSION'])
|
|
52
|
-
parts = [*version_num.split('.'), '0', '0'][:3]
|
|
53
|
-
parts[0] = re.match(r'\d+', parts[0]).group()
|
|
54
|
-
self.pg_config_version = reduce(lambda v, x: v * 100 + int(x), parts, 0)
|
|
55
|
-
|
|
56
|
-
os.environ['LANGUAGE'] = 'en' # set default locale language to en. All messages will use this locale
|
|
57
|
-
test_env = os.environ.copy()
|
|
58
|
-
envs_list = [
|
|
59
|
-
'LANGUAGE',
|
|
60
|
-
'LC_ALL',
|
|
61
|
-
'PGCONNECT_TIMEOUT',
|
|
62
|
-
'PGDATA',
|
|
63
|
-
'PGDATABASE',
|
|
64
|
-
'PGHOSTADDR',
|
|
65
|
-
'PGREQUIRESSL',
|
|
66
|
-
'PGSERVICE',
|
|
67
|
-
'PGSSLMODE',
|
|
68
|
-
'PGUSER',
|
|
69
|
-
'PGPORT',
|
|
70
|
-
'PGHOST'
|
|
71
|
-
]
|
|
72
|
-
|
|
73
|
-
for e in envs_list:
|
|
74
|
-
test_env.pop(e, None)
|
|
75
|
-
|
|
76
|
-
test_env['LC_MESSAGES'] = 'C'
|
|
77
|
-
test_env['LC_TIME'] = 'C'
|
|
78
|
-
self._test_env = test_env
|
|
79
|
-
|
|
80
|
-
# Get the directory from which the script was executed
|
|
81
|
-
self.source_path = os.getcwd()
|
|
82
|
-
tmp_path = test_env.get('PGPROBACKUP_TMP_DIR')
|
|
83
|
-
if tmp_path and os.path.isabs(tmp_path):
|
|
84
|
-
self.tmp_path = tmp_path
|
|
85
|
-
else:
|
|
86
|
-
self.tmp_path = os.path.abspath(
|
|
87
|
-
os.path.join(self.source_path, tmp_path or os.path.join('tests', 'tmp_dirs'))
|
|
88
|
-
)
|
|
89
|
-
|
|
90
|
-
os.makedirs(self.tmp_path, exist_ok=True)
|
|
91
|
-
|
|
92
|
-
self.username = getpass.getuser()
|
|
93
|
-
|
|
94
|
-
self.probackup_path = None
|
|
95
|
-
if 'PGPROBACKUPBIN' in test_env:
|
|
96
|
-
if shutil.which(test_env["PGPROBACKUPBIN"]):
|
|
97
|
-
self.probackup_path = test_env["PGPROBACKUPBIN"]
|
|
98
|
-
else:
|
|
99
|
-
raise Exception(
|
|
100
|
-
'pg_probackup binary not found at PGPROBACKUPBIN={0}'.format(
|
|
101
|
-
test_env["PGPROBACKUPBIN"]))
|
|
102
|
-
|
|
103
|
-
if not self.probackup_path:
|
|
104
|
-
probackup_path_tmp = os.path.join(
|
|
105
|
-
testgres.get_pg_config()['BINDIR'], 'pg_probackup')
|
|
106
|
-
|
|
107
|
-
if os.path.isfile(probackup_path_tmp):
|
|
108
|
-
if not os.access(probackup_path_tmp, os.X_OK):
|
|
109
|
-
logging.warning('{0} is not an executable file'.format(
|
|
110
|
-
probackup_path_tmp))
|
|
111
|
-
else:
|
|
112
|
-
self.probackup_path = probackup_path_tmp
|
|
113
|
-
|
|
114
|
-
if not self.probackup_path:
|
|
115
|
-
probackup_path_tmp = self.source_path
|
|
116
|
-
|
|
117
|
-
if os.path.isfile(probackup_path_tmp):
|
|
118
|
-
if not os.access(probackup_path_tmp, os.X_OK):
|
|
119
|
-
logging.warning('{0} is not an executable file'.format(
|
|
120
|
-
probackup_path_tmp))
|
|
121
|
-
else:
|
|
122
|
-
self.probackup_path = probackup_path_tmp
|
|
123
|
-
|
|
124
|
-
if not self.probackup_path:
|
|
125
|
-
raise Exception('pg_probackup binary is not found')
|
|
126
|
-
|
|
127
|
-
if os.name == 'posix':
|
|
128
|
-
self.EXTERNAL_DIRECTORY_DELIMITER = ':'
|
|
129
|
-
os.environ['PATH'] = os.path.dirname(
|
|
130
|
-
self.probackup_path) + ':' + os.environ['PATH']
|
|
131
|
-
|
|
132
|
-
elif os.name == 'nt':
|
|
133
|
-
self.EXTERNAL_DIRECTORY_DELIMITER = ';'
|
|
134
|
-
os.environ['PATH'] = os.path.dirname(
|
|
135
|
-
self.probackup_path) + ';' + os.environ['PATH']
|
|
136
|
-
|
|
137
|
-
self.probackup_old_path = None
|
|
138
|
-
if 'PGPROBACKUPBIN_OLD' in test_env:
|
|
139
|
-
if (os.path.isfile(test_env['PGPROBACKUPBIN_OLD']) and os.access(test_env['PGPROBACKUPBIN_OLD'], os.X_OK)):
|
|
140
|
-
self.probackup_old_path = test_env['PGPROBACKUPBIN_OLD']
|
|
141
|
-
else:
|
|
142
|
-
if self.verbose:
|
|
143
|
-
print('PGPROBACKUPBIN_OLD is not an executable file')
|
|
144
|
-
|
|
145
|
-
self.probackup_version = None
|
|
146
|
-
self.old_probackup_version = None
|
|
147
|
-
|
|
148
|
-
probackup_version_output = subprocess.check_output(
|
|
149
|
-
[self.probackup_path, "--version"],
|
|
150
|
-
stderr=subprocess.STDOUT,
|
|
151
|
-
).decode('utf-8')
|
|
152
|
-
match = re.search(r"\d+\.\d+\.\d+",
|
|
153
|
-
probackup_version_output)
|
|
154
|
-
self.probackup_version = match.group(0) if match else None
|
|
155
|
-
match = re.search(r"\(compressions: ([^)]*)\)", probackup_version_output)
|
|
156
|
-
compressions = match.group(1) if match else None
|
|
157
|
-
if compressions:
|
|
158
|
-
self.probackup_compressions = {s.strip() for s in compressions.split(',')}
|
|
159
|
-
else:
|
|
160
|
-
self.probackup_compressions = []
|
|
161
|
-
|
|
162
|
-
if self.probackup_old_path:
|
|
163
|
-
old_probackup_version_output = subprocess.check_output(
|
|
164
|
-
[self.probackup_old_path, "--version"],
|
|
165
|
-
stderr=subprocess.STDOUT,
|
|
166
|
-
).decode('utf-8')
|
|
167
|
-
match = re.search(r"\d+\.\d+\.\d+",
|
|
168
|
-
old_probackup_version_output)
|
|
169
|
-
self.old_probackup_version = match.group(0) if match else None
|
|
170
|
-
|
|
171
|
-
self.remote = test_env.get('PGPROBACKUP_SSH_REMOTE', None) == 'ON'
|
|
172
|
-
self.ptrack = test_env.get('PG_PROBACKUP_PTRACK', None) == 'ON' and self.pg_config_version >= 110000
|
|
173
|
-
self.wal_tree_enabled = test_env.get('PG_PROBACKUP_WAL_TREE_ENABLED', None) == 'ON'
|
|
174
|
-
|
|
175
|
-
self.bckp_source = test_env.get('PG_PROBACKUP_SOURCE', 'pro').lower()
|
|
176
|
-
if self.bckp_source not in ('base', 'direct', 'pro'):
|
|
177
|
-
raise Exception("Wrong PG_PROBACKUP_SOURCE value. Available options: base|direct|pro")
|
|
178
|
-
|
|
179
|
-
self.paranoia = test_env.get('PG_PROBACKUP_PARANOIA', None) == 'ON'
|
|
180
|
-
env_compress = test_env.get('ARCHIVE_COMPRESSION', None)
|
|
181
|
-
if env_compress:
|
|
182
|
-
env_compress = env_compress.lower()
|
|
183
|
-
if env_compress in ('on', 'zlib'):
|
|
184
|
-
self.compress_suffix = '.gz'
|
|
185
|
-
self.archive_compress = 'zlib'
|
|
186
|
-
elif env_compress == 'lz4':
|
|
187
|
-
if not HAVE_LZ4:
|
|
188
|
-
raise LZ4_error
|
|
189
|
-
if 'lz4' not in self.probackup_compressions:
|
|
190
|
-
raise Exception("pg_probackup is not compiled with lz4 support")
|
|
191
|
-
self.compress_suffix = '.lz4'
|
|
192
|
-
self.archive_compress = 'lz4'
|
|
193
|
-
elif env_compress == 'zstd':
|
|
194
|
-
if not HAVE_ZSTD:
|
|
195
|
-
raise ZSTD_error
|
|
196
|
-
if 'zstd' not in self.probackup_compressions:
|
|
197
|
-
raise Exception("pg_probackup is not compiled with zstd support")
|
|
198
|
-
self.compress_suffix = '.zst'
|
|
199
|
-
self.archive_compress = 'zstd'
|
|
200
|
-
else:
|
|
201
|
-
self.compress_suffix = ''
|
|
202
|
-
self.archive_compress = False
|
|
203
|
-
|
|
204
|
-
cfs_compress = test_env.get('PG_PROBACKUP_CFS_COMPRESS', None)
|
|
205
|
-
if cfs_compress:
|
|
206
|
-
self.cfs_compress = cfs_compress.lower()
|
|
207
|
-
else:
|
|
208
|
-
self.cfs_compress = self.archive_compress
|
|
209
|
-
|
|
210
|
-
os.environ["PGAPPNAME"] = "pg_probackup"
|
|
211
|
-
self.delete_logs = delete_logs
|
|
212
|
-
|
|
213
|
-
if self.probackup_version.split('.')[0].isdigit():
|
|
214
|
-
self.major_version = int(self.probackup_version.split('.')[0])
|
|
215
|
-
else:
|
|
216
|
-
raise Exception('Can\'t process pg_probackup version \"{}\": the major version is expected to be a number'.format(self.probackup_version))
|
|
217
|
-
|
|
218
|
-
self.valgrind = test_env.get('PG_PROBACKUP_VALGRIND')
|
|
219
|
-
self.valgrind_sup_path = test_env.get('PG_PROBACKUP_VALGRIND_SUP')
|
|
220
|
-
|
|
221
|
-
def test_env(self):
|
|
222
|
-
return self._test_env.copy()
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
try:
|
|
226
|
-
init_params = Init()
|
|
227
|
-
except Exception as e:
|
|
228
|
-
logging.error(str(e))
|
|
229
|
-
logging.warning("testgres.plugins.probackup2.init_params is set to None.")
|
|
230
|
-
init_params = None
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{testgres_pg_probackup2-0.8.2 → testgres_pg_probackup2-0.9.1}/pg_probackup2/storage/fs_backup.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|