xdump 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xdump/__init__.py +1 -0
- xdump/base.py +286 -0
- xdump/cli/__init__.py +0 -0
- xdump/cli/base.py +28 -0
- xdump/cli/dump.py +159 -0
- xdump/cli/load.py +75 -0
- xdump/cli/utils.py +32 -0
- xdump/extra/__init__.py +0 -0
- xdump/extra/django/__init__.py +0 -0
- xdump/extra/django/apps.py +5 -0
- xdump/extra/django/xdump/__init__.py +0 -0
- xdump/extra/django/xdump/management/__init__.py +0 -0
- xdump/extra/django/xdump/management/commands/__init__.py +0 -0
- xdump/extra/django/xdump/management/commands/xdump.py +37 -0
- xdump/extra/django/xdump/management/commands/xload.py +31 -0
- xdump/extra/django/xdump/management/core.py +81 -0
- xdump/logging.py +19 -0
- xdump/postgresql.py +222 -0
- xdump/py.typed +0 -0
- xdump/sqlite.py +157 -0
- xdump/utils.py +7 -0
- xdump-1.0.0.dist-info/METADATA +241 -0
- xdump-1.0.0.dist-info/RECORD +26 -0
- xdump-1.0.0.dist-info/WHEEL +4 -0
- xdump-1.0.0.dist-info/entry_points.txt +3 -0
- xdump-1.0.0.dist-info/licenses/LICENSE +21 -0
xdump/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "1.0.0"
|
xdump/base.py
ADDED
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import os
|
|
3
|
+
import zipfile
|
|
4
|
+
from collections.abc import Iterator, Sequence
|
|
5
|
+
from contextlib import contextmanager
|
|
6
|
+
from time import time
|
|
7
|
+
from typing import IO, Any
|
|
8
|
+
|
|
9
|
+
from .logging import get_logger
|
|
10
|
+
|
|
11
|
+
Row = dict[str, Any]
|
|
12
|
+
ForeignKey = dict[str, str]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class BaseBackend:
|
|
16
|
+
connections: dict[str, dict[str, Any]] = {"default": {}}
|
|
17
|
+
schema_filename = "dump/schema.sql"
|
|
18
|
+
initial_setup_files: tuple[str, ...] = (schema_filename,)
|
|
19
|
+
data_dir = "dump/data/"
|
|
20
|
+
dbname: str
|
|
21
|
+
verbosity: int
|
|
22
|
+
|
|
23
|
+
@property
|
|
24
|
+
def logger(self) -> logging.Logger:
|
|
25
|
+
if not hasattr(self, "_logger"):
|
|
26
|
+
self._logger = get_logger("XDump", self.verbosity)
|
|
27
|
+
return self._logger
|
|
28
|
+
|
|
29
|
+
@contextmanager
|
|
30
|
+
def log_query(self, sql: str | bytes, params: Any = None) -> Iterator[None]:
|
|
31
|
+
self.logger.debug("Execute query: %s", sql)
|
|
32
|
+
self.logger.debug("Parameters: %s", str(params))
|
|
33
|
+
with self.log_time():
|
|
34
|
+
yield
|
|
35
|
+
|
|
36
|
+
@contextmanager
|
|
37
|
+
def log_time(self, message: str = "Execution time: %s") -> Iterator[None]:
|
|
38
|
+
start = time()
|
|
39
|
+
yield
|
|
40
|
+
self.logger.debug(message, (time() - start))
|
|
41
|
+
|
|
42
|
+
# Connection
|
|
43
|
+
|
|
44
|
+
@property
|
|
45
|
+
def _open_connections(self) -> dict[str, Any]:
|
|
46
|
+
connections: dict[str, Any] = self.__dict__.setdefault("_open_connections_cache", {})
|
|
47
|
+
return connections
|
|
48
|
+
|
|
49
|
+
@property
|
|
50
|
+
def _open_cursors(self) -> dict[str, Any]:
|
|
51
|
+
cursors: dict[str, Any] = self.__dict__.setdefault("_open_cursors_cache", {})
|
|
52
|
+
return cursors
|
|
53
|
+
|
|
54
|
+
def get_connection(self, name: str = "default") -> Any:
|
|
55
|
+
if name not in self._open_connections:
|
|
56
|
+
self._open_connections[name] = self.connect(**self.connections[name])
|
|
57
|
+
return self._open_connections[name]
|
|
58
|
+
|
|
59
|
+
def get_cursor(self, name: str = "default") -> Any:
|
|
60
|
+
if name not in self._open_cursors:
|
|
61
|
+
self._open_cursors[name] = self.get_connection(name).cursor()
|
|
62
|
+
return self._open_cursors[name]
|
|
63
|
+
|
|
64
|
+
def connect(self, **kwargs: Any) -> Any:
|
|
65
|
+
"""Create a connection to the database."""
|
|
66
|
+
raise NotImplementedError
|
|
67
|
+
|
|
68
|
+
def get_connection_kwargs(self, **kwargs: Any) -> dict[str, Any]:
|
|
69
|
+
for option in ("dbname", "user", "password", "host", "port"):
|
|
70
|
+
kwargs.setdefault(option, getattr(self, option))
|
|
71
|
+
return kwargs
|
|
72
|
+
|
|
73
|
+
def cache_clear(self) -> None:
|
|
74
|
+
"""Closes all open connections."""
|
|
75
|
+
self._open_cursors.clear()
|
|
76
|
+
for connection in self._open_connections.values():
|
|
77
|
+
connection.close()
|
|
78
|
+
self._open_connections.clear()
|
|
79
|
+
|
|
80
|
+
# Low-level commands executors
|
|
81
|
+
|
|
82
|
+
def run_dump(self, *args: str) -> bytes:
|
|
83
|
+
"""Runs built-in utility for data / schema dumping."""
|
|
84
|
+
raise NotImplementedError
|
|
85
|
+
|
|
86
|
+
def run(self, sql: str | bytes, params: Any = None, using: str = "default") -> list[Row]:
|
|
87
|
+
with self.log_query(sql, params):
|
|
88
|
+
cursor = self.get_cursor(using)
|
|
89
|
+
cursor.execute(sql, params)
|
|
90
|
+
if cursor.description is None:
|
|
91
|
+
# The statement returns no rows, e.g. `BEGIN` or `INSERT`
|
|
92
|
+
return []
|
|
93
|
+
rows: list[Row] = cursor.fetchall()
|
|
94
|
+
return rows
|
|
95
|
+
|
|
96
|
+
@contextmanager
|
|
97
|
+
def transaction(self) -> Iterator[None]:
|
|
98
|
+
"""Runs block of code inside a transaction."""
|
|
99
|
+
self.run("BEGIN")
|
|
100
|
+
yield
|
|
101
|
+
self.run("COMMIT")
|
|
102
|
+
|
|
103
|
+
# Dumping the data
|
|
104
|
+
|
|
105
|
+
def dump(
|
|
106
|
+
self,
|
|
107
|
+
filename: str,
|
|
108
|
+
full_tables: Sequence[str] = (),
|
|
109
|
+
partial_tables: dict[str, str] | None = None,
|
|
110
|
+
compression: int = zipfile.ZIP_DEFLATED,
|
|
111
|
+
dump_schema: bool = True,
|
|
112
|
+
dump_data: bool = True,
|
|
113
|
+
) -> None:
|
|
114
|
+
"""Creates a dump, which could be used to restore the database."""
|
|
115
|
+
self.input_check(full_tables, partial_tables)
|
|
116
|
+
with self.log_time("Total execution time: %s"):
|
|
117
|
+
partial_tables = partial_tables or {}
|
|
118
|
+
with zipfile.ZipFile(filename, "w", compression) as file:
|
|
119
|
+
if dump_schema:
|
|
120
|
+
self.write_initial_setup(file)
|
|
121
|
+
if dump_data:
|
|
122
|
+
self.add_related_data(full_tables, partial_tables)
|
|
123
|
+
self.write_full_tables(file, full_tables)
|
|
124
|
+
self.write_partial_tables(file, partial_tables)
|
|
125
|
+
|
|
126
|
+
def input_check(self, full_tables: Sequence[str], partial_tables: dict[str, str] | None) -> None:
|
|
127
|
+
if full_tables and partial_tables:
|
|
128
|
+
common_tables = set(full_tables) & set(partial_tables)
|
|
129
|
+
if common_tables:
|
|
130
|
+
raise ValueError(
|
|
131
|
+
"`partial_tables` should not contain tables from `full_tables`. Common tables: {tables}".format(
|
|
132
|
+
tables=", ".join(common_tables)
|
|
133
|
+
)
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
def add_related_data(self, full_tables: Sequence[str], partial_tables: dict[str, str]) -> None:
|
|
137
|
+
"""Updates selects for partial tables to grab all objects, that are referenced by full / partial tables."""
|
|
138
|
+
tables = self.get_tables_for_related_data(full_tables, partial_tables)
|
|
139
|
+
for table in tables:
|
|
140
|
+
self.update_partial_tables(table, full_tables, partial_tables)
|
|
141
|
+
|
|
142
|
+
def get_tables_for_related_data(self, full_tables: Sequence[str], partial_tables: dict[str, str]) -> Sequence[str]:
|
|
143
|
+
return tuple(full_tables) + tuple(partial_tables.keys())
|
|
144
|
+
|
|
145
|
+
def update_partial_tables(self, table: str, full_tables: Sequence[str], partial_tables: dict[str, str]) -> None:
|
|
146
|
+
self.update_recursive_relations(table, full_tables, partial_tables)
|
|
147
|
+
self.update_non_recursive_relations(table, full_tables, partial_tables)
|
|
148
|
+
|
|
149
|
+
def update_recursive_relations(
|
|
150
|
+
self, table: str, full_tables: Sequence[str], partial_tables: dict[str, str]
|
|
151
|
+
) -> None:
|
|
152
|
+
for foreign_key in self.get_foreign_keys(table, full_tables, recursive=True):
|
|
153
|
+
if table in partial_tables:
|
|
154
|
+
partial_tables[table] = RECURSIVE_QUERY_TEMPLATE.format(
|
|
155
|
+
source=partial_tables[table], target=table, **foreign_key
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
def update_non_recursive_relations(
|
|
159
|
+
self, table: str, full_tables: Sequence[str], partial_tables: dict[str, str]
|
|
160
|
+
) -> None:
|
|
161
|
+
for foreign_key in self.get_foreign_keys(table, full_tables):
|
|
162
|
+
sql = self.get_related_data_sql(foreign_key, full_tables, partial_tables)
|
|
163
|
+
if sql:
|
|
164
|
+
foreign_table = foreign_key["foreign_table_name"]
|
|
165
|
+
if foreign_table in partial_tables:
|
|
166
|
+
partial_tables[foreign_table] += " UNION " + sql
|
|
167
|
+
else:
|
|
168
|
+
partial_tables[foreign_table] = sql
|
|
169
|
+
# Now we select more than before for given table, so we have to do check related data for it.
|
|
170
|
+
self.update_partial_tables(foreign_table, full_tables, partial_tables)
|
|
171
|
+
|
|
172
|
+
def get_foreign_keys(
|
|
173
|
+
self, table: str, full_tables: Sequence[str] = (), recursive: bool = False
|
|
174
|
+
) -> Iterator[ForeignKey]:
|
|
175
|
+
"""Looks for foreign keys in the given table. Excluding ones, that will be dumped in ``full_tables``."""
|
|
176
|
+
raise NotImplementedError
|
|
177
|
+
|
|
178
|
+
def get_related_data_sql(
|
|
179
|
+
self, foreign_key: ForeignKey, full_tables: Sequence[str], partial_tables: dict[str, str]
|
|
180
|
+
) -> str | None:
|
|
181
|
+
"""Generates SQL to select related data, that is referred from another table."""
|
|
182
|
+
table_name = foreign_key["table_name"]
|
|
183
|
+
if table_name in full_tables:
|
|
184
|
+
source = foreign_key["table_name"]
|
|
185
|
+
elif table_name in partial_tables:
|
|
186
|
+
source = f"({partial_tables[table_name]}) T"
|
|
187
|
+
else:
|
|
188
|
+
return None
|
|
189
|
+
return """
|
|
190
|
+
SELECT
|
|
191
|
+
*
|
|
192
|
+
FROM {foreign_table_name}
|
|
193
|
+
WHERE {foreign_column_name} IN (
|
|
194
|
+
SELECT {column_name} FROM {source}
|
|
195
|
+
)""".format(source=source, **foreign_key)
|
|
196
|
+
|
|
197
|
+
def write_initial_setup(self, file: zipfile.ZipFile) -> None:
|
|
198
|
+
self.write_schema(file)
|
|
199
|
+
|
|
200
|
+
def write_schema(self, file: zipfile.ZipFile) -> None:
|
|
201
|
+
"""Writes a DB schema, functions, etc to the archive."""
|
|
202
|
+
schema = self.dump_schema()
|
|
203
|
+
file.writestr(self.schema_filename, schema)
|
|
204
|
+
|
|
205
|
+
def dump_schema(self) -> bytes:
|
|
206
|
+
raise NotImplementedError
|
|
207
|
+
|
|
208
|
+
def write_full_tables(self, file: zipfile.ZipFile, tables: Sequence[str]) -> None:
|
|
209
|
+
"""Writes a complete tables dump to the archive."""
|
|
210
|
+
for table_name in tables:
|
|
211
|
+
self.write_data_file(file, table_name, f"SELECT * FROM {table_name}")
|
|
212
|
+
|
|
213
|
+
def write_partial_tables(self, file: zipfile.ZipFile, config: dict[str, str]) -> None:
|
|
214
|
+
for table_name, sql in config.items():
|
|
215
|
+
self.write_data_file(file, table_name, sql)
|
|
216
|
+
|
|
217
|
+
def write_data_file(self, file: zipfile.ZipFile, table_name: str, sql: str) -> None:
|
|
218
|
+
data = self.export_to_csv(sql)
|
|
219
|
+
file.writestr(f"{self.data_dir}{table_name}.csv", data)
|
|
220
|
+
|
|
221
|
+
def export_to_csv(self, sql: str) -> bytes:
|
|
222
|
+
raise NotImplementedError
|
|
223
|
+
|
|
224
|
+
# Database re-creation
|
|
225
|
+
|
|
226
|
+
def recreate_database(self, owner: str | None = None) -> None:
|
|
227
|
+
"""Drops all connections to the database, drops the database and creates it again."""
|
|
228
|
+
self.drop_database(self.dbname)
|
|
229
|
+
self.create_database(self.dbname, owner)
|
|
230
|
+
self.cache_clear()
|
|
231
|
+
|
|
232
|
+
def drop_database(self, dbname: str) -> None:
|
|
233
|
+
raise NotImplementedError
|
|
234
|
+
|
|
235
|
+
def create_database(self, dbname: str, owner: str | None = None) -> None:
|
|
236
|
+
raise NotImplementedError
|
|
237
|
+
|
|
238
|
+
def truncate(self) -> None:
|
|
239
|
+
"""Truncates all tables in the DB. Alternative for the re-creation option."""
|
|
240
|
+
raise NotImplementedError
|
|
241
|
+
|
|
242
|
+
# Loading the dump
|
|
243
|
+
|
|
244
|
+
def load(self, filename: str) -> None:
|
|
245
|
+
"""Loads schema, sequences and data into the database."""
|
|
246
|
+
with self.log_time("Total execution time: %s"):
|
|
247
|
+
archive = zipfile.ZipFile(filename)
|
|
248
|
+
self.initial_setup(archive)
|
|
249
|
+
self.load_data(archive)
|
|
250
|
+
|
|
251
|
+
def initial_setup(self, archive: zipfile.ZipFile) -> None:
|
|
252
|
+
"""Loads schema and initial database configuration."""
|
|
253
|
+
name_list = archive.namelist()
|
|
254
|
+
for filename in self.initial_setup_files:
|
|
255
|
+
if filename not in name_list:
|
|
256
|
+
continue
|
|
257
|
+
sql = archive.read(filename)
|
|
258
|
+
self.run_setup_file(sql)
|
|
259
|
+
|
|
260
|
+
def run_setup_file(self, sql: bytes) -> None:
|
|
261
|
+
self.run(sql)
|
|
262
|
+
|
|
263
|
+
def load_data(self, archive: zipfile.ZipFile) -> None:
|
|
264
|
+
"""Loads all data from data files inside the archive to the database."""
|
|
265
|
+
with self.transaction():
|
|
266
|
+
for name in archive.namelist():
|
|
267
|
+
if name.startswith(self.data_dir):
|
|
268
|
+
fd = archive.open(name)
|
|
269
|
+
table_name = os.path.basename(name).split(".")[0]
|
|
270
|
+
self.load_data_file(table_name, fd)
|
|
271
|
+
|
|
272
|
+
def load_data_file(self, table_name: str, fd: IO[bytes]) -> None:
|
|
273
|
+
"""Loads a data file into the database."""
|
|
274
|
+
raise NotImplementedError
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
RECURSIVE_QUERY_TEMPLATE = """
|
|
278
|
+
WITH RECURSIVE recursive_cte AS (
|
|
279
|
+
SELECT * FROM ({source}) S
|
|
280
|
+
UNION
|
|
281
|
+
SELECT T.*
|
|
282
|
+
FROM {table_name} T
|
|
283
|
+
INNER JOIN recursive_cte ON (recursive_cte.{column_name} = T.{foreign_column_name})
|
|
284
|
+
)
|
|
285
|
+
SELECT * FROM recursive_cte
|
|
286
|
+
"""
|
xdump/cli/__init__.py
ADDED
|
File without changes
|
xdump/cli/base.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import click
|
|
2
|
+
|
|
3
|
+
from .utils import Decorator
|
|
4
|
+
|
|
5
|
+
COMMON_DECORATORS: list[Decorator] = [
|
|
6
|
+
click.option("-D", "--dbname", required=True, help="database to work with"),
|
|
7
|
+
click.option(
|
|
8
|
+
"-v",
|
|
9
|
+
"--verbosity",
|
|
10
|
+
help="verbosity level",
|
|
11
|
+
default=0,
|
|
12
|
+
count=True,
|
|
13
|
+
type=click.IntRange(0, 2),
|
|
14
|
+
),
|
|
15
|
+
]
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
PG_DECORATORS: list[Decorator] = [
|
|
19
|
+
click.option("-U", "--user", required=True, help="connect as specified database user"),
|
|
20
|
+
click.option("-W", "--password", help="password for the DB connection"),
|
|
21
|
+
click.option(
|
|
22
|
+
"-H",
|
|
23
|
+
"--host",
|
|
24
|
+
default="127.0.0.1",
|
|
25
|
+
help="database server host or socket directory",
|
|
26
|
+
),
|
|
27
|
+
click.option("-P", "--port", default="5432", help="database server port number"),
|
|
28
|
+
]
|
xdump/cli/dump.py
ADDED
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import zipfile
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
import click
|
|
5
|
+
|
|
6
|
+
from .base import COMMON_DECORATORS, PG_DECORATORS
|
|
7
|
+
from .utils import Decorator, apply_decorators, init_backend
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@click.group(name="xdump")
|
|
11
|
+
@click.version_option()
|
|
12
|
+
def dump() -> None:
|
|
13
|
+
pass
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def parse_partial(ctx: click.Context, param: click.Parameter, value: tuple[str, ...]) -> dict[str, str]:
|
|
17
|
+
"""Parse values for `partial` option. They should be in the format `table:select SQL`."""
|
|
18
|
+
|
|
19
|
+
def parse_value(value: str) -> tuple[str, str]:
|
|
20
|
+
try:
|
|
21
|
+
table_name, sql = value.split(":", 1)
|
|
22
|
+
return table_name.strip(), sql.strip()
|
|
23
|
+
except ValueError:
|
|
24
|
+
raise click.exceptions.BadParameter(
|
|
25
|
+
'partial table specification should be in the following format: "table:select SQL"',
|
|
26
|
+
param=param,
|
|
27
|
+
) from None
|
|
28
|
+
|
|
29
|
+
return dict(parse_value(partial) for partial in value)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
COMPRESSION_MAPPING = {
|
|
33
|
+
"deflated": zipfile.ZIP_DEFLATED,
|
|
34
|
+
"stored": zipfile.ZIP_STORED,
|
|
35
|
+
"bzip2": zipfile.ZIP_BZIP2,
|
|
36
|
+
"lzma": zipfile.ZIP_LZMA,
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
DEFAULT_PARAMETERS: list[Decorator] = [
|
|
41
|
+
dump.command(),
|
|
42
|
+
click.option("-o", "--output", required=True, help="output file name"),
|
|
43
|
+
click.option(
|
|
44
|
+
"-f",
|
|
45
|
+
"--full",
|
|
46
|
+
help="table name to be fully dumped. Could be used multiple times",
|
|
47
|
+
multiple=True,
|
|
48
|
+
),
|
|
49
|
+
click.option(
|
|
50
|
+
"-p",
|
|
51
|
+
"--partial",
|
|
52
|
+
help='partial tables specification in a form "table_name:select SQL". Could be used multiple times',
|
|
53
|
+
callback=parse_partial,
|
|
54
|
+
multiple=True,
|
|
55
|
+
),
|
|
56
|
+
click.option(
|
|
57
|
+
"-c",
|
|
58
|
+
"--compression",
|
|
59
|
+
help="dump compression level",
|
|
60
|
+
default="deflated",
|
|
61
|
+
type=click.Choice(list(COMPRESSION_MAPPING.keys())),
|
|
62
|
+
),
|
|
63
|
+
click.option(
|
|
64
|
+
"--schema/--no-schema",
|
|
65
|
+
help="include / exclude the schema from the dump",
|
|
66
|
+
default=True,
|
|
67
|
+
),
|
|
68
|
+
click.option(
|
|
69
|
+
"--data/--no-data",
|
|
70
|
+
help="include / exclude the data from the dump",
|
|
71
|
+
default=True,
|
|
72
|
+
),
|
|
73
|
+
] + COMMON_DECORATORS
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def base_dump(
|
|
77
|
+
backend_path: str,
|
|
78
|
+
output: str,
|
|
79
|
+
full: tuple[str, ...],
|
|
80
|
+
partial: dict[str, str],
|
|
81
|
+
compression: str,
|
|
82
|
+
schema: bool,
|
|
83
|
+
data: bool,
|
|
84
|
+
**kwargs: Any,
|
|
85
|
+
) -> None:
|
|
86
|
+
"""Common implementation of dump command. Writes a few logs, imports a backend and makes a dump."""
|
|
87
|
+
|
|
88
|
+
click.echo("Dumping ...")
|
|
89
|
+
click.echo(f"Output file: {output}")
|
|
90
|
+
|
|
91
|
+
backend = init_backend(backend_path, **kwargs)
|
|
92
|
+
try:
|
|
93
|
+
backend.dump(
|
|
94
|
+
output,
|
|
95
|
+
full_tables=full,
|
|
96
|
+
partial_tables=partial,
|
|
97
|
+
compression=COMPRESSION_MAPPING[compression],
|
|
98
|
+
dump_schema=schema,
|
|
99
|
+
dump_data=data,
|
|
100
|
+
)
|
|
101
|
+
finally:
|
|
102
|
+
backend.cache_clear()
|
|
103
|
+
click.echo("Done!")
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
@apply_decorators(DEFAULT_PARAMETERS + PG_DECORATORS)
|
|
107
|
+
def postgres(
|
|
108
|
+
user: str,
|
|
109
|
+
password: str | None,
|
|
110
|
+
host: str,
|
|
111
|
+
port: str,
|
|
112
|
+
dbname: str,
|
|
113
|
+
verbosity: int,
|
|
114
|
+
output: str,
|
|
115
|
+
full: tuple[str, ...],
|
|
116
|
+
partial: dict[str, str],
|
|
117
|
+
compression: str,
|
|
118
|
+
schema: bool,
|
|
119
|
+
data: bool,
|
|
120
|
+
) -> None:
|
|
121
|
+
base_dump(
|
|
122
|
+
"xdump.postgresql.PostgreSQLBackend",
|
|
123
|
+
output,
|
|
124
|
+
full,
|
|
125
|
+
partial,
|
|
126
|
+
compression,
|
|
127
|
+
schema,
|
|
128
|
+
data,
|
|
129
|
+
user=user,
|
|
130
|
+
password=password,
|
|
131
|
+
host=host,
|
|
132
|
+
port=port,
|
|
133
|
+
dbname=dbname,
|
|
134
|
+
verbosity=verbosity,
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
@apply_decorators(DEFAULT_PARAMETERS)
|
|
139
|
+
def sqlite(
|
|
140
|
+
dbname: str,
|
|
141
|
+
verbosity: int,
|
|
142
|
+
output: str,
|
|
143
|
+
full: tuple[str, ...],
|
|
144
|
+
partial: dict[str, str],
|
|
145
|
+
compression: str,
|
|
146
|
+
schema: bool,
|
|
147
|
+
data: bool,
|
|
148
|
+
) -> None:
|
|
149
|
+
base_dump(
|
|
150
|
+
"xdump.sqlite.SQLiteBackend",
|
|
151
|
+
output,
|
|
152
|
+
full,
|
|
153
|
+
partial,
|
|
154
|
+
compression,
|
|
155
|
+
schema,
|
|
156
|
+
data,
|
|
157
|
+
dbname=dbname,
|
|
158
|
+
verbosity=verbosity,
|
|
159
|
+
)
|
xdump/cli/load.py
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
from typing import Any
|
|
2
|
+
|
|
3
|
+
import click
|
|
4
|
+
|
|
5
|
+
from .base import COMMON_DECORATORS, PG_DECORATORS
|
|
6
|
+
from .utils import Decorator, apply_decorators, init_backend
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@click.group(name="xload")
|
|
10
|
+
@click.version_option()
|
|
11
|
+
def load() -> None:
|
|
12
|
+
pass
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
DEFAULT_PARAMETERS: list[Decorator] = [
|
|
16
|
+
load.command(),
|
|
17
|
+
click.option("-i", "--input", required=True, help="input file name"),
|
|
18
|
+
click.option(
|
|
19
|
+
"-m",
|
|
20
|
+
"--cleanup-method",
|
|
21
|
+
help="method of DB cleaning up",
|
|
22
|
+
type=click.Choice(("recreate", "truncate")),
|
|
23
|
+
),
|
|
24
|
+
] + COMMON_DECORATORS
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def base_load(backend_path: str, input: str, cleanup_method: str | None, **kwargs: Any) -> None:
|
|
28
|
+
click.echo("Loading ...")
|
|
29
|
+
click.echo(f"Input file: {input}")
|
|
30
|
+
|
|
31
|
+
backend = init_backend(backend_path, **kwargs)
|
|
32
|
+
try:
|
|
33
|
+
if cleanup_method == "truncate":
|
|
34
|
+
backend.truncate()
|
|
35
|
+
elif cleanup_method == "recreate":
|
|
36
|
+
backend.recreate_database()
|
|
37
|
+
backend.load(input)
|
|
38
|
+
finally:
|
|
39
|
+
backend.cache_clear()
|
|
40
|
+
click.echo("Done!")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@apply_decorators(DEFAULT_PARAMETERS + PG_DECORATORS)
|
|
44
|
+
def postgres(
|
|
45
|
+
user: str,
|
|
46
|
+
password: str | None,
|
|
47
|
+
host: str,
|
|
48
|
+
port: str,
|
|
49
|
+
dbname: str,
|
|
50
|
+
verbosity: int,
|
|
51
|
+
input: str,
|
|
52
|
+
cleanup_method: str | None,
|
|
53
|
+
) -> None:
|
|
54
|
+
base_load(
|
|
55
|
+
"xdump.postgresql.PostgreSQLBackend",
|
|
56
|
+
input,
|
|
57
|
+
cleanup_method,
|
|
58
|
+
user=user,
|
|
59
|
+
password=password,
|
|
60
|
+
host=host,
|
|
61
|
+
port=port,
|
|
62
|
+
dbname=dbname,
|
|
63
|
+
verbosity=verbosity,
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@apply_decorators(DEFAULT_PARAMETERS)
|
|
68
|
+
def sqlite(dbname: str, verbosity: int, input: str, cleanup_method: str | None) -> None:
|
|
69
|
+
base_load(
|
|
70
|
+
"xdump.sqlite.SQLiteBackend",
|
|
71
|
+
input,
|
|
72
|
+
cleanup_method,
|
|
73
|
+
dbname=dbname,
|
|
74
|
+
verbosity=verbosity,
|
|
75
|
+
)
|
xdump/cli/utils.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
from collections.abc import Callable, Sequence
|
|
2
|
+
from typing import Any, TypeVar
|
|
3
|
+
|
|
4
|
+
from ..base import BaseBackend
|
|
5
|
+
|
|
6
|
+
F = TypeVar("F", bound=Callable[..., Any])
|
|
7
|
+
Decorator = Callable[[Any], Any]
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def apply_decorators(decorators: Sequence[Decorator]) -> Callable[[F], F]:
|
|
11
|
+
"""Apply multiple decorators to the same function. Useful for reusing common decorators among many functions."""
|
|
12
|
+
|
|
13
|
+
def wrapper(func: F) -> F:
|
|
14
|
+
for decorator in reversed(decorators):
|
|
15
|
+
func = decorator(func)
|
|
16
|
+
return func
|
|
17
|
+
|
|
18
|
+
return wrapper
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def import_backend(path: str) -> type[BaseBackend]:
|
|
22
|
+
"""Import a backend class by dotted path."""
|
|
23
|
+
module_name, class_name = path.rsplit(".", 1)
|
|
24
|
+
module = __import__(module_name, fromlist=[class_name])
|
|
25
|
+
backend_class: type[BaseBackend] = getattr(module, class_name)
|
|
26
|
+
return backend_class
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def init_backend(backend_path: str, **kwargs: Any) -> BaseBackend:
|
|
30
|
+
"""Initialize a DB backend."""
|
|
31
|
+
backend_class = import_backend(backend_path)
|
|
32
|
+
return backend_class(**kwargs)
|
xdump/extra/__init__.py
ADDED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
from typing import Any
|
|
2
|
+
|
|
3
|
+
from django.core.management.base import CommandParser
|
|
4
|
+
|
|
5
|
+
from xdump.base import BaseBackend
|
|
6
|
+
|
|
7
|
+
from ..core import XDumpCommand
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Command(XDumpCommand):
|
|
11
|
+
help = "Creates an SQL dump with latest data."
|
|
12
|
+
|
|
13
|
+
def add_arguments(self, parser: CommandParser) -> None:
|
|
14
|
+
super().add_arguments(parser)
|
|
15
|
+
parser.add_argument(
|
|
16
|
+
"-d",
|
|
17
|
+
"--dump-data",
|
|
18
|
+
action="store",
|
|
19
|
+
dest="dump_data",
|
|
20
|
+
help="Control if the data should be dumped.",
|
|
21
|
+
required=False,
|
|
22
|
+
default=True,
|
|
23
|
+
)
|
|
24
|
+
parser.add_argument(
|
|
25
|
+
"-s",
|
|
26
|
+
"--dump-schema",
|
|
27
|
+
action="store",
|
|
28
|
+
dest="dump_schema",
|
|
29
|
+
help="Control if the schema should be dumped.",
|
|
30
|
+
required=False,
|
|
31
|
+
default=True,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
def _handle(self, filename: str, backend: BaseBackend, **options: Any) -> None:
|
|
35
|
+
backend.dump(
|
|
36
|
+
filename, dump_data=options["dump_data"], dump_schema=options["dump_schema"], **self.get_dump_kwargs()
|
|
37
|
+
)
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
from typing import Any
|
|
2
|
+
|
|
3
|
+
from django.core.management.base import CommandParser
|
|
4
|
+
|
|
5
|
+
from xdump.base import BaseBackend
|
|
6
|
+
|
|
7
|
+
from ..core import XDumpCommand
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Command(XDumpCommand):
|
|
11
|
+
help = "Loads an SQL dump."
|
|
12
|
+
|
|
13
|
+
def add_arguments(self, parser: CommandParser) -> None:
|
|
14
|
+
super().add_arguments(parser)
|
|
15
|
+
parser.add_argument(
|
|
16
|
+
"-m",
|
|
17
|
+
"--cleanup-method",
|
|
18
|
+
action="store",
|
|
19
|
+
nargs="?",
|
|
20
|
+
choices=["recreate", "truncate"],
|
|
21
|
+
dest="cleanup_method",
|
|
22
|
+
help="Method of DB cleaning up",
|
|
23
|
+
required=False,
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
def _handle(self, filename: str, backend: BaseBackend, **options: Any) -> None:
|
|
27
|
+
if options["cleanup_method"] == "truncate":
|
|
28
|
+
backend.truncate()
|
|
29
|
+
elif options["cleanup_method"] == "recreate":
|
|
30
|
+
backend.recreate_database()
|
|
31
|
+
backend.load(filename)
|