spindb 0.68.7 → 0.68.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/create.js +4 -1
- package/dist/cli/commands/create.js.map +1 -1
- package/dist/cli/commands/menu/backup-handlers.js +8 -2
- package/dist/cli/commands/menu/backup-handlers.js.map +1 -1
- package/dist/cli/commands/restore.js +11 -1
- package/dist/cli/commands/restore.js.map +1 -1
- package/dist/config/version.js +1 -1
- package/dist/core/mysql-family-binary-resolver.js +84 -0
- package/dist/core/mysql-family-binary-resolver.js.map +1 -0
- package/dist/core/pull-manager.js +12 -2
- package/dist/core/pull-manager.js.map +1 -1
- package/dist/core/server-handshake.js +250 -0
- package/dist/core/server-handshake.js.map +1 -0
- package/dist/engines/mysql/backup.js +13 -29
- package/dist/engines/mysql/backup.js.map +1 -1
- package/dist/engines/mysql/dump-normalize.js +201 -0
- package/dist/engines/mysql/dump-normalize.js.map +1 -0
- package/dist/engines/mysql/index.js +214 -10
- package/dist/engines/mysql/index.js.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MariaDB -> MySQL Dump Normalization
|
|
3
|
+
*
|
|
4
|
+
* A dump taken from a MariaDB source with `mariadb-dump` is valid SQL for
|
|
5
|
+
* MariaDB, not for MySQL. Three things in it stop a MySQL 8 or 9 target, and
|
|
6
|
+
* all three are mechanical rewrites of statements MySQL has an exact or a
|
|
7
|
+
* near-exact equivalent for:
|
|
8
|
+
*
|
|
9
|
+
* 1. **uca1400 collations.** MariaDB 11.4 made `utf8mb4_uca1400_ai_ci` the
|
|
10
|
+
* default database collation, so nearly every `CREATE TABLE` in a modern
|
|
11
|
+
* MariaDB dump names a collation MySQL has never had. MySQL's UCA 9.0.0
|
|
12
|
+
* collations are the same Unicode collation algorithm at a newer UCA
|
|
13
|
+
* version, so `utf8mb4_0900_ai_ci` is the honest counterpart.
|
|
14
|
+
* 2. **`NO_AUTO_CREATE_USER` in `SET sql_mode`.** MariaDB still emits it around
|
|
15
|
+
* triggers and routines; MySQL removed the mode in 8.0 and answers
|
|
16
|
+
* ERROR 1231 (Variable 'sql_mode' can't be set to the value of
|
|
17
|
+
* 'NO_AUTO_CREATE_USER').
|
|
18
|
+
* 3. **The sandbox directive.** `mariadb-dump` 11.x opens every dump with
|
|
19
|
+
* `/*M!999999\- enable the sandbox mode *\/`. MySQL reads it as a comment
|
|
20
|
+
* and is unharmed, but it is MariaDB-only noise in a file we are converting.
|
|
21
|
+
*
|
|
22
|
+
* What is deliberately NOT rewritten: sequences (`CREATE SEQUENCE`, `nextval()`
|
|
23
|
+
* defaults) and MariaDB-only column types (`UUID`, `INET4`, `INET6`,
|
|
24
|
+
* `VECTOR`). MySQL has no equivalent, and quietly substituting one would put
|
|
25
|
+
* different data in the target than the source holds. Those statements are left
|
|
26
|
+
* to fail with the server's own error, which names the object that could not be
|
|
27
|
+
* converted. `json_valid()` CHECK constraints need no rewrite: MySQL has
|
|
28
|
+
* `json_valid()`.
|
|
29
|
+
*
|
|
30
|
+
* Every rule is line-oriented, so a multi-gigabyte dump is rewritten as a
|
|
31
|
+
* stream and never held in memory.
|
|
32
|
+
*/
|
|
33
|
+
import { createReadStream, createWriteStream } from 'fs';
|
|
34
|
+
import { createInterface } from 'readline';
|
|
35
|
+
import { once } from 'events';
|
|
36
|
+
export function emptyNormalizationCounts() {
|
|
37
|
+
return {
|
|
38
|
+
collationsMapped: 0,
|
|
39
|
+
sqlModeFlagsRemoved: 0,
|
|
40
|
+
sandboxDirectivesDropped: 0,
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
export function totalRewrites(counts) {
|
|
44
|
+
return (counts.collationsMapped +
|
|
45
|
+
counts.sqlModeFlagsRemoved +
|
|
46
|
+
counts.sandboxDirectivesDropped);
|
|
47
|
+
}
|
|
48
|
+
// MySQL's UCA 9.0.0 counterparts, keyed by MariaDB's accent/case suffix.
|
|
49
|
+
// MySQL has no accent-insensitive-but-case-sensitive utf8mb4 collation, so
|
|
50
|
+
// `ai_cs` maps to `as_cs`: the case sensitivity the schema asked for is kept,
|
|
51
|
+
// and accent sensitivity is the property that cannot be honored.
|
|
52
|
+
const UCA1400_SUFFIX_MAP = {
|
|
53
|
+
ai_ci: 'utf8mb4_0900_ai_ci',
|
|
54
|
+
as_cs: 'utf8mb4_0900_as_cs',
|
|
55
|
+
as_ci: 'utf8mb4_0900_as_ci',
|
|
56
|
+
ai_cs: 'utf8mb4_0900_as_cs',
|
|
57
|
+
};
|
|
58
|
+
// The two suffixes that ask for case sensitivity. MariaDB spells accent and
|
|
59
|
+
// case sensitivity separately; only the case half survives a utf8mb3 target.
|
|
60
|
+
const CASE_SENSITIVE_SUFFIXES = new Set(['as_cs', 'ai_cs']);
|
|
61
|
+
const UTF8MB4_FALLBACK = 'utf8mb4_0900_ai_ci';
|
|
62
|
+
const UTF8MB3_FALLBACK = 'utf8mb3_unicode_ci';
|
|
63
|
+
// MySQL has no utf8mb3 UCA collation that is case sensitive, so a schema that
|
|
64
|
+
// asked for one gets `utf8mb3_bin`. Binary ordering is not UCA ordering, but it
|
|
65
|
+
// is the only utf8mb3 collation that keeps case sensitivity, and losing case
|
|
66
|
+
// sensitivity silently changes which rows a comparison matches.
|
|
67
|
+
const UTF8MB3_CASE_SENSITIVE = 'utf8mb3_bin';
|
|
68
|
+
const UTF8MB4_UCA1400 = /\butf8mb4_uca1400_([a-z0-9_]+)/gi;
|
|
69
|
+
const UTF8MB3_UCA1400 = /\b(?:utf8mb3|utf8)_uca1400_([a-z0-9_]+)/gi;
|
|
70
|
+
const SQL_MODE_ASSIGNMENT = /\bsql_mode\s*=\s*'/i;
|
|
71
|
+
const ROW_STATEMENT = /^\s*(?:INSERT|REPLACE)\b/i;
|
|
72
|
+
const SANDBOX_DIRECTIVE = /^\s*\/\*M!\d+\\?-.*sandbox mode.*\*\/\s*;?\s*$/i;
|
|
73
|
+
const NO_AUTO_CREATE_USER = 'NO_AUTO_CREATE_USER';
|
|
74
|
+
/**
|
|
75
|
+
* Map one MariaDB uca1400 collation name to its MySQL counterpart.
|
|
76
|
+
*
|
|
77
|
+
* `nopad_` is a padding variant MySQL does not spell out in the collation
|
|
78
|
+
* name, so it is stripped and the remaining suffix decides the target. Any
|
|
79
|
+
* suffix that is not one of the four accent/case combinations (locale-specific
|
|
80
|
+
* collations such as `utf8mb4_uca1400_swedish_ai_ci`) falls back to the
|
|
81
|
+
* default `utf8mb4_0900_ai_ci`, because MySQL's locale-tailored collations do
|
|
82
|
+
* not line up one-for-one with MariaDB's.
|
|
83
|
+
*/
|
|
84
|
+
export function mapUca1400Collation(suffix) {
|
|
85
|
+
return UCA1400_SUFFIX_MAP[normalizeUca1400Suffix(suffix)] ?? UTF8MB4_FALLBACK;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Map one MariaDB utf8mb3 uca1400 collation name to a utf8mb3 collation MySQL
|
|
89
|
+
* has.
|
|
90
|
+
*
|
|
91
|
+
* Same suffix parsing as `mapUca1400Collation`, different target set: MySQL's
|
|
92
|
+
* UCA 9.0.0 collations are utf8mb4 only, so a utf8mb3 column cannot follow the
|
|
93
|
+
* charset it declared into `utf8mb4_0900_*`. What it can keep is case
|
|
94
|
+
* sensitivity, so `_as_cs` and `_ai_cs` (and their `nopad_` forms) map to
|
|
95
|
+
* `utf8mb3_bin` rather than being flattened into a `_ci` collation that would
|
|
96
|
+
* quietly start matching rows the source did not.
|
|
97
|
+
*/
|
|
98
|
+
export function mapUca1400Utf8mb3Collation(suffix) {
|
|
99
|
+
return CASE_SENSITIVE_SUFFIXES.has(normalizeUca1400Suffix(suffix))
|
|
100
|
+
? UTF8MB3_CASE_SENSITIVE
|
|
101
|
+
: UTF8MB3_FALLBACK;
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* `nopad_` is a padding variant MySQL does not spell out in the collation name,
|
|
105
|
+
* so it is stripped before the accent/case suffix is read.
|
|
106
|
+
*/
|
|
107
|
+
function normalizeUca1400Suffix(suffix) {
|
|
108
|
+
return suffix.toLowerCase().replace(/^nopad_/, '');
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Rewrite one line of a MariaDB dump so a MySQL server accepts it.
|
|
112
|
+
*
|
|
113
|
+
* Pure: the whole contract of the conversion lives here, so it is unit tested
|
|
114
|
+
* line by line rather than by diffing a real dump.
|
|
115
|
+
*/
|
|
116
|
+
export function normalizeMariaDbDumpForMysql(line) {
|
|
117
|
+
const counts = emptyNormalizationCounts();
|
|
118
|
+
if (SANDBOX_DIRECTIVE.test(line)) {
|
|
119
|
+
counts.sandboxDirectivesDropped = 1;
|
|
120
|
+
return { line: null, counts };
|
|
121
|
+
}
|
|
122
|
+
// A row is data, never DDL. A collation name is an ordinary string that a
|
|
123
|
+
// user is entitled to store (a migration log, a schema-tracking table, an
|
|
124
|
+
// ORM's own bookkeeping), and rewriting it would put different bytes in the
|
|
125
|
+
// target than the source holds - the exact outcome the sql_mode rule below
|
|
126
|
+
// already refuses. No INSERT or REPLACE in a dump carries a collation that
|
|
127
|
+
// needs converting, so the whole line is left alone.
|
|
128
|
+
if (ROW_STATEMENT.test(line)) {
|
|
129
|
+
return { line, counts };
|
|
130
|
+
}
|
|
131
|
+
let result = line.replace(UTF8MB4_UCA1400, (_match, suffix) => {
|
|
132
|
+
counts.collationsMapped++;
|
|
133
|
+
return mapUca1400Collation(suffix);
|
|
134
|
+
});
|
|
135
|
+
result = result.replace(UTF8MB3_UCA1400, (_match, suffix) => {
|
|
136
|
+
counts.collationsMapped++;
|
|
137
|
+
return mapUca1400Utf8mb3Collation(suffix);
|
|
138
|
+
});
|
|
139
|
+
if (result.includes(NO_AUTO_CREATE_USER) &&
|
|
140
|
+
SQL_MODE_ASSIGNMENT.test(result)) {
|
|
141
|
+
result = result.replace(/'([^']*)'/g, (match, contents) => {
|
|
142
|
+
if (!contents.includes(NO_AUTO_CREATE_USER))
|
|
143
|
+
return match;
|
|
144
|
+
const kept = contents
|
|
145
|
+
.split(',')
|
|
146
|
+
.filter((mode) => mode.trim() !== NO_AUTO_CREATE_USER);
|
|
147
|
+
const removed = contents.split(',').length - kept.length;
|
|
148
|
+
if (removed === 0)
|
|
149
|
+
return match;
|
|
150
|
+
counts.sqlModeFlagsRemoved += removed;
|
|
151
|
+
return `'${kept.join(',')}'`;
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
return { line: result, counts };
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* Stream a MariaDB dump through `normalizeMariaDbDumpForMysql`, writing the
|
|
158
|
+
* MySQL-ready result to a new file.
|
|
159
|
+
*
|
|
160
|
+
* Line by line with explicit backpressure: a dump is routinely larger than the
|
|
161
|
+
* process can hold, so it is never read into a string.
|
|
162
|
+
*
|
|
163
|
+
* **Read and written as `latin1`, never `utf8`.** A dump is not guaranteed to
|
|
164
|
+
* be valid UTF-8: a `latin1` column, or a BLOB that `mariadb-dump` writes as
|
|
165
|
+
* escaped bytes rather than a hex literal, puts arbitrary bytes above 0x7F in
|
|
166
|
+
* the file. Decoding those as UTF-8 replaces every invalid sequence with
|
|
167
|
+
* U+FFFD, so the conversion silently rewrote the user's data (a `0x80 0xFF`
|
|
168
|
+
* BLOB came out as six `EF BF BD` bytes). `latin1` maps bytes 1:1 to code
|
|
169
|
+
* points 0-255 and back, so the file round-trips byte for byte and only the
|
|
170
|
+
* ASCII tokens the rules match are ever changed. Line splitting stays correct
|
|
171
|
+
* because MySQL escapes `\n` and `\r` inside string literals, so a raw
|
|
172
|
+
* newline byte never appears in dump data.
|
|
173
|
+
*/
|
|
174
|
+
export async function normalizeMariaDbDumpFile(options) {
|
|
175
|
+
const { inputPath, outputPath } = options;
|
|
176
|
+
const counts = emptyNormalizationCounts();
|
|
177
|
+
const input = createReadStream(inputPath, { encoding: 'latin1' });
|
|
178
|
+
const output = createWriteStream(outputPath, { encoding: 'latin1' });
|
|
179
|
+
const lines = createInterface({ input, crlfDelay: Infinity });
|
|
180
|
+
try {
|
|
181
|
+
for await (const line of lines) {
|
|
182
|
+
const normalized = normalizeMariaDbDumpForMysql(line);
|
|
183
|
+
counts.collationsMapped += normalized.counts.collationsMapped;
|
|
184
|
+
counts.sqlModeFlagsRemoved += normalized.counts.sqlModeFlagsRemoved;
|
|
185
|
+
counts.sandboxDirectivesDropped +=
|
|
186
|
+
normalized.counts.sandboxDirectivesDropped;
|
|
187
|
+
if (normalized.line === null)
|
|
188
|
+
continue;
|
|
189
|
+
if (!output.write(`${normalized.line}\n`)) {
|
|
190
|
+
await once(output, 'drain');
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
finally {
|
|
195
|
+
lines.close();
|
|
196
|
+
output.end();
|
|
197
|
+
}
|
|
198
|
+
await once(output, 'close');
|
|
199
|
+
return counts;
|
|
200
|
+
}
|
|
201
|
+
//# sourceMappingURL=dump-normalize.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dump-normalize.js","sourceRoot":"","sources":["../../../engines/mysql/dump-normalize.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,IAAI,CAAA;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAA;AAC1C,OAAO,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAA;AAkB7B,MAAM,UAAU,wBAAwB;IACtC,OAAO;QACL,gBAAgB,EAAE,CAAC;QACnB,mBAAmB,EAAE,CAAC;QACtB,wBAAwB,EAAE,CAAC;KAC5B,CAAA;AACH,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,MAA+B;IAC3D,OAAO,CACL,MAAM,CAAC,gBAAgB;QACvB,MAAM,CAAC,mBAAmB;QAC1B,MAAM,CAAC,wBAAwB,CAChC,CAAA;AACH,CAAC;AAED,yEAAyE;AACzE,2EAA2E;AAC3E,8EAA8E;AAC9E,iEAAiE;AACjE,MAAM,kBAAkB,GAA2B;IACjD,KAAK,EAAE,oBAAoB;IAC3B,KAAK,EAAE,oBAAoB;IAC3B,KAAK,EAAE,oBAAoB;IAC3B,KAAK,EAAE,oBAAoB;CAC5B,CAAA;AAED,4EAA4E;AAC5E,6EAA6E;AAC7E,MAAM,uBAAuB,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAA;AAE3D,MAAM,gBAAgB,GAAG,oBAAoB,CAAA;AAC7C,MAAM,gBAAgB,GAAG,oBAAoB,CAAA;AAC7C,8EAA8E;AAC9E,gFAAgF;AAChF,6EAA6E;AAC7E,gEAAgE;AAChE,MAAM,sBAAsB,GAAG,aAAa,CAAA;AAE5C,MAAM,eAAe,GAAG,kCAAkC,CAAA;AAC1D,MAAM,eAAe,GAAG,2CAA2C,CAAA;AACnE,MAAM,mBAAmB,GAAG,qBAAqB,CAAA;AACjD,MAAM,aAAa,GAAG,2BAA2B,CAAA;AACjD,MAAM,iBAAiB,GAAG,iDAAiD,CAAA;AAC3E,MAAM,mBAAmB,GAAG,qBAAqB,CAAA;AAEjD;;;;;;;;;GASG;AACH,MAAM,UAAU,mBAAmB,CAAC,MAAc;IAChD,OAAO,kBAAkB,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC,IAAI,gBAAgB,CAAA;AAC/E,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,0BAA0B,CAAC,MAAc;IACvD,OAAO,uBAAuB,CAAC,GAAG,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC;QAChE,CAAC,CAAC,sBAAsB;QACxB,CAAC,CAAC,gBAAgB,CAAA;AACtB,CAAC;AAED;;;GAGG;AACH,SAAS,sBAAsB,CAAC,MAAc;IAC5C,OAAO,MAAM,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAA;AACpD,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,4BAA4B,CAAC,IAAY;IACvD,MAAM,MAAM,GAAG,wBAAwB,EAAE,CAAA;IAEzC,IAAI,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACjC,MAAM,CAAC,wBAAwB,GAAG,CAAC,CAAA;QACnC,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,CAAA;IAC/B,CAAC;IAED,0EAA0E;IAC1E,0EAA0E;IAC1E,4EAA4E;IAC5E,2EAA2E;IAC3E,2EAA2E;IAC3E,qDAAqD;IACrD,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC7B,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAA;IACzB,CAAC;IAED,IAAI,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,MAAM,EAAE,MAAc,EAAE,EAAE;QACpE,MAAM,CAAC,gBAAgB,EAAE,CAAA;QACzB,OAAO,mBAAmB,CAAC,MAAM,CAAC,CAAA;IACpC,CAAC,CAAC,CAAA;IAEF,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,MAAM,EAAE,MAAc,EAAE,EAAE;QAClE,MAAM,CAAC,gBAAgB,EAAE,CAAA;QACzB,OAAO,0BAA0B,CAAC,MAAM,CAAC,CAAA;IAC3C,CAAC,CAAC,CAAA;IAEF,IACE,MAAM,CAAC,QAAQ,CAAC,mBAAmB,CAAC;QACpC,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,EAChC,CAAC;QACD,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,KAAK,EAAE,QAAgB,EAAE,EAAE;YAChE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,mBAAmB,CAAC;gBAAE,OAAO,KAAK,CAAA;YAEzD,MAAM,IAAI,GAAG,QAAQ;iBAClB,KAAK,CAAC,GAAG,CAAC;iBACV,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,mBAAmB,CAAC,CAAA;YACxD,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAA;YACxD,IAAI,OAAO,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAA;YAE/B,MAAM,CAAC,mBAAmB,IAAI,OAAO,CAAA;YACrC,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAA;QAC9B,CAAC,CAAC,CAAA;IACJ,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,CAAA;AACjC,CAAC;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAAC,OAG9C;IACC,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,OAAO,CAAA;IACzC,MAAM,MAAM,GAAG,wBAAwB,EAAE,CAAA;IAEzC,MAAM,KAAK,GAAG,gBAAgB,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAA;IACjE,MAAM,MAAM,GAAG,iBAAiB,CAAC,UAAU,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAA;IACpE,MAAM,KAAK,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAA;IAE7D,IAAI,CAAC;QACH,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC/B,MAAM,UAAU,GAAG,4BAA4B,CAAC,IAAI,CAAC,CAAA;YACrD,MAAM,CAAC,gBAAgB,IAAI,UAAU,CAAC,MAAM,CAAC,gBAAgB,CAAA;YAC7D,MAAM,CAAC,mBAAmB,IAAI,UAAU,CAAC,MAAM,CAAC,mBAAmB,CAAA;YACnE,MAAM,CAAC,wBAAwB;gBAC7B,UAAU,CAAC,MAAM,CAAC,wBAAwB,CAAA;YAE5C,IAAI,UAAU,CAAC,IAAI,KAAK,IAAI;gBAAE,SAAQ;YACtC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,UAAU,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC;gBAC1C,MAAM,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;YAC7B,CAAC;QACH,CAAC;IACH,CAAC;YAAS,CAAC;QACT,KAAK,CAAC,KAAK,EAAE,CAAA;QACb,MAAM,CAAC,GAAG,EAAE,CAAA;IACd,CAAC;IAED,MAAM,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAC3B,OAAO,MAAM,CAAA;AACf,CAAC"}
|
|
@@ -22,6 +22,11 @@ import { SUPPORTED_MAJOR_VERSIONS, FALLBACK_VERSION_MAP } from './version-maps.j
|
|
|
22
22
|
import { resolveEngineVersion } from '../../core/version-resolver.js';
|
|
23
23
|
import { detectBackupFormat as detectBackupFormatImpl, restoreBackup, parseConnectionString, } from './restore.js';
|
|
24
24
|
import { createBackup } from './backup.js';
|
|
25
|
+
import { normalizeMariaDbDumpFile, totalRewrites, } from './dump-normalize.js';
|
|
26
|
+
import { buildMariaDbRemoteDumpArgs } from '../mariadb/index.js';
|
|
27
|
+
import { mariadbBinaryManager } from '../mariadb/binary-manager.js';
|
|
28
|
+
import { probeMysqlFamilyServer, } from '../../core/server-handshake.js';
|
|
29
|
+
import { resolveBundledMysqlFamilyBinary } from '../../core/mysql-family-binary-resolver.js';
|
|
25
30
|
import { Engine, Platform, } from '../../types/index.js';
|
|
26
31
|
import { parseTSVToQueryResult } from '../../core/query-parser.js';
|
|
27
32
|
const execFileAsync = promisify(execFile);
|
|
@@ -837,9 +842,49 @@ export class MySQLEngine extends BaseEngine {
|
|
|
837
842
|
return null;
|
|
838
843
|
}
|
|
839
844
|
}
|
|
845
|
+
/**
|
|
846
|
+
* Dump a remote database reached through a `mysql://` URL.
|
|
847
|
+
*
|
|
848
|
+
* **The dump tool follows the SOURCE server, not this container.** That is
|
|
849
|
+
* the rule PostgreSQL already follows (`validateDumpCompatibility` in
|
|
850
|
+
* `engines/postgresql/version-validator.ts` swaps in a `pg_dump` that can
|
|
851
|
+
* read the remote major). PostgreSQL only has to follow a version. The MySQL
|
|
852
|
+
* family has to follow a flavor as well, because MySQL and MariaDB share a
|
|
853
|
+
* wire protocol and a URL scheme: `mysql://host/db` says nothing about which
|
|
854
|
+
* server is on the other end, and the two ship different, mutually
|
|
855
|
+
* unusable dump tools.
|
|
856
|
+
*
|
|
857
|
+
* Guessing wrong is not a degraded dump, it is no dump at all. `mysqldump` 9
|
|
858
|
+
* cannot even authenticate against a MariaDB server: MySQL 9 dropped the
|
|
859
|
+
* `mysql_native_password` client plugin, MariaDB's root user still uses it,
|
|
860
|
+
* and the dump dies on `Authentication plugin 'mysql_native_password' cannot
|
|
861
|
+
* be loaded` before it reads a single table. No flag fixes that.
|
|
862
|
+
*
|
|
863
|
+
* So the source is probed first (`core/server-handshake.ts` reads the
|
|
864
|
+
* greeting the server sends on connect, without writing anything or
|
|
865
|
+
* authenticating), and a MariaDB source is dumped with `mariadb-dump` and
|
|
866
|
+
* then normalized for this MySQL target. A MySQL source, or a source that
|
|
867
|
+
* could not be probed, takes the unchanged `mysqldump` path.
|
|
868
|
+
*/
|
|
840
869
|
async dumpFromConnectionString(connectionString, outputPath, options) {
|
|
841
|
-
const dumpPath = await this.getDumpPath();
|
|
842
870
|
const { host, port, user, password, database } = parseConnectionString(connectionString);
|
|
871
|
+
const source = await probeMysqlFamilyServer({
|
|
872
|
+
host,
|
|
873
|
+
port: parseInt(port, 10) || 3306,
|
|
874
|
+
});
|
|
875
|
+
if (source.flavor === 'mariadb') {
|
|
876
|
+
return this.dumpFromMariaDbSource({
|
|
877
|
+
source,
|
|
878
|
+
host,
|
|
879
|
+
port,
|
|
880
|
+
user,
|
|
881
|
+
password,
|
|
882
|
+
database,
|
|
883
|
+
outputPath,
|
|
884
|
+
excludeTables: options?.excludeTables,
|
|
885
|
+
});
|
|
886
|
+
}
|
|
887
|
+
const dumpPath = await this.getDumpPath(options?.targetVersion);
|
|
843
888
|
const args = buildMysqlRemoteDumpArgs({
|
|
844
889
|
host,
|
|
845
890
|
port,
|
|
@@ -848,12 +893,159 @@ export class MySQLEngine extends BaseEngine {
|
|
|
848
893
|
outputPath,
|
|
849
894
|
excludeTables: options?.excludeTables,
|
|
850
895
|
});
|
|
896
|
+
const { stdout, stderr } = await this.runDumpTool({
|
|
897
|
+
toolPath: dumpPath,
|
|
898
|
+
toolName: 'mysqldump',
|
|
899
|
+
args,
|
|
900
|
+
password,
|
|
901
|
+
});
|
|
902
|
+
logDebug('Remote dump taken with mysqldump', {
|
|
903
|
+
host,
|
|
904
|
+
sourceFlavor: source.flavor,
|
|
905
|
+
sourceVersion: source.version,
|
|
906
|
+
dumpPath,
|
|
907
|
+
});
|
|
908
|
+
return {
|
|
909
|
+
filePath: outputPath,
|
|
910
|
+
stdout,
|
|
911
|
+
stderr,
|
|
912
|
+
code: 0,
|
|
913
|
+
remoteSource: {
|
|
914
|
+
flavor: source.flavor,
|
|
915
|
+
serverVersion: source.version || undefined,
|
|
916
|
+
dumpTool: 'mysqldump',
|
|
917
|
+
},
|
|
918
|
+
};
|
|
919
|
+
}
|
|
920
|
+
/**
|
|
921
|
+
* Dump a MariaDB source into a file this MySQL container can restore.
|
|
922
|
+
*
|
|
923
|
+
* Two steps, both of which are needed: take the dump with MariaDB's own
|
|
924
|
+
* tool, then rewrite the MariaDB-only SQL it emits (see
|
|
925
|
+
* `engines/mysql/dump-normalize.ts` for exactly which statements and why).
|
|
926
|
+
*/
|
|
927
|
+
async dumpFromMariaDbSource(options) {
|
|
928
|
+
const { source, host, port, user, password, database, outputPath, excludeTables, } = options;
|
|
929
|
+
const tool = await this.resolveMariaDbDumpPath(source);
|
|
930
|
+
// mariadb-dump writes through --result-file, so the raw dump lands beside
|
|
931
|
+
// the requested path and is rewritten into it.
|
|
932
|
+
const rawPath = `${outputPath}.mariadb`;
|
|
933
|
+
const warnings = [
|
|
934
|
+
`Source is MariaDB ${source.version}; dumped with mariadb-dump ${tool.version} ` +
|
|
935
|
+
'(mysqldump cannot authenticate against a MariaDB server)',
|
|
936
|
+
];
|
|
937
|
+
if (tool.downloaded) {
|
|
938
|
+
warnings.push(`Downloaded MariaDB ${tool.version} client tools to read the source`);
|
|
939
|
+
}
|
|
940
|
+
let counts;
|
|
941
|
+
let stdout;
|
|
942
|
+
let stderr;
|
|
943
|
+
try {
|
|
944
|
+
const dump = await this.runDumpTool({
|
|
945
|
+
toolPath: tool.path,
|
|
946
|
+
toolName: 'mariadb-dump',
|
|
947
|
+
args: buildMariaDbRemoteDumpArgs({
|
|
948
|
+
host,
|
|
949
|
+
port,
|
|
950
|
+
user,
|
|
951
|
+
database,
|
|
952
|
+
outputPath: rawPath,
|
|
953
|
+
excludeTables,
|
|
954
|
+
}),
|
|
955
|
+
password,
|
|
956
|
+
});
|
|
957
|
+
stdout = dump.stdout;
|
|
958
|
+
stderr = dump.stderr;
|
|
959
|
+
counts = await normalizeMariaDbDumpFile({
|
|
960
|
+
inputPath: rawPath,
|
|
961
|
+
outputPath,
|
|
962
|
+
});
|
|
963
|
+
}
|
|
964
|
+
finally {
|
|
965
|
+
await rm(rawPath, { force: true });
|
|
966
|
+
}
|
|
967
|
+
if (totalRewrites(counts) > 0) {
|
|
968
|
+
warnings.push(`Converted the dump for MySQL: ${counts.collationsMapped} uca1400 collations mapped, ` +
|
|
969
|
+
`${counts.sqlModeFlagsRemoved} NO_AUTO_CREATE_USER sql_mode flags removed, ` +
|
|
970
|
+
`${counts.sandboxDirectivesDropped} MariaDB sandbox directives dropped`);
|
|
971
|
+
}
|
|
972
|
+
warnings.push('MariaDB-only objects (sequences, UUID/INET4/INET6/VECTOR columns) are not ' +
|
|
973
|
+
'converted and will be reported by the server if the source uses them');
|
|
974
|
+
logDebug('Remote dump taken with mariadb-dump', {
|
|
975
|
+
host,
|
|
976
|
+
sourceVersion: source.version,
|
|
977
|
+
dumpPath: tool.path,
|
|
978
|
+
...counts,
|
|
979
|
+
});
|
|
980
|
+
return {
|
|
981
|
+
filePath: outputPath,
|
|
982
|
+
stdout,
|
|
983
|
+
stderr,
|
|
984
|
+
code: 0,
|
|
985
|
+
warnings,
|
|
986
|
+
remoteSource: {
|
|
987
|
+
flavor: 'mariadb',
|
|
988
|
+
serverVersion: source.version,
|
|
989
|
+
dumpTool: 'mariadb-dump',
|
|
990
|
+
dumpToolVersion: tool.version,
|
|
991
|
+
rewrites: {
|
|
992
|
+
collationsMapped: counts.collationsMapped,
|
|
993
|
+
sqlModeFlagsRemoved: counts.sqlModeFlagsRemoved,
|
|
994
|
+
sandboxDirectivesDropped: counts.sandboxDirectivesDropped,
|
|
995
|
+
},
|
|
996
|
+
},
|
|
997
|
+
};
|
|
998
|
+
}
|
|
999
|
+
/**
|
|
1000
|
+
* Find a `mariadb-dump` able to read this source, downloading MariaDB's
|
|
1001
|
+
* client tools if none is installed.
|
|
1002
|
+
*
|
|
1003
|
+
* The source's own major.minor line is preferred, so an 11.8 server is read
|
|
1004
|
+
* by an 11.8 tool when one is on disk; otherwise the newest installed
|
|
1005
|
+
* MariaDB is used, and only a machine with no MariaDB at all downloads.
|
|
1006
|
+
*/
|
|
1007
|
+
async resolveMariaDbDumpPath(source) {
|
|
1008
|
+
const preferVersion = source.majorVersion !== null && source.minorVersion !== null
|
|
1009
|
+
? `${source.majorVersion}.${source.minorVersion}`
|
|
1010
|
+
: undefined;
|
|
1011
|
+
const installed = resolveBundledMysqlFamilyBinary({
|
|
1012
|
+
engine: 'mariadb',
|
|
1013
|
+
tool: 'mariadb-dump',
|
|
1014
|
+
preferVersion,
|
|
1015
|
+
});
|
|
1016
|
+
if (installed)
|
|
1017
|
+
return { ...installed, downloaded: false };
|
|
1018
|
+
const { platform: p, arch: a } = this.getPlatformInfo();
|
|
1019
|
+
const defaultVersion = getEngineDefaults('mariadb').defaultVersion;
|
|
1020
|
+
try {
|
|
1021
|
+
await mariadbBinaryManager.ensureInstalled(defaultVersion, p, a);
|
|
1022
|
+
}
|
|
1023
|
+
catch (error) {
|
|
1024
|
+
throw new SpinDBError(ErrorCodes.DEPENDENCY_MISSING, `The source server is MariaDB ${source.version}, which needs mariadb-dump, ` +
|
|
1025
|
+
`and downloading MariaDB ${defaultVersion} failed: ${error.message}`, 'fatal', 'Download MariaDB client tools: spindb engines download mariadb');
|
|
1026
|
+
}
|
|
1027
|
+
const downloaded = resolveBundledMysqlFamilyBinary({
|
|
1028
|
+
engine: 'mariadb',
|
|
1029
|
+
tool: 'mariadb-dump',
|
|
1030
|
+
preferVersion,
|
|
1031
|
+
});
|
|
1032
|
+
if (!downloaded) {
|
|
1033
|
+
throw new SpinDBError(ErrorCodes.DEPENDENCY_MISSING, `The source server is MariaDB ${source.version}, but no mariadb-dump is installed.`, 'fatal', 'Download MariaDB client tools: spindb engines download mariadb');
|
|
1034
|
+
}
|
|
1035
|
+
return { ...downloaded, downloaded: true };
|
|
1036
|
+
}
|
|
1037
|
+
/**
|
|
1038
|
+
* Run a dump tool and collect its output. Shared by both source flavors so
|
|
1039
|
+
* the failure surface (and the password handling) is identical.
|
|
1040
|
+
*/
|
|
1041
|
+
async runDumpTool(options) {
|
|
1042
|
+
const { toolPath, toolName, args, password } = options;
|
|
851
1043
|
const spawnOptions = {
|
|
852
1044
|
stdio: ['pipe', 'pipe', 'pipe'],
|
|
853
1045
|
env: password ? { ...process.env, MYSQL_PWD: password } : process.env,
|
|
854
1046
|
};
|
|
855
1047
|
return new Promise((resolve, reject) => {
|
|
856
|
-
const proc = spawn(
|
|
1048
|
+
const proc = spawn(toolPath, args, spawnOptions);
|
|
857
1049
|
let stdout = '';
|
|
858
1050
|
let stderr = '';
|
|
859
1051
|
proc.stdout?.on('data', (data) => {
|
|
@@ -865,20 +1057,32 @@ export class MySQLEngine extends BaseEngine {
|
|
|
865
1057
|
proc.on('error', reject);
|
|
866
1058
|
proc.on('close', (code) => {
|
|
867
1059
|
if (code === 0) {
|
|
868
|
-
resolve({
|
|
869
|
-
filePath: outputPath,
|
|
870
|
-
stdout,
|
|
871
|
-
stderr,
|
|
872
|
-
code,
|
|
873
|
-
});
|
|
1060
|
+
resolve({ stdout, stderr });
|
|
874
1061
|
}
|
|
875
1062
|
else {
|
|
876
|
-
reject(new Error(stderr ||
|
|
1063
|
+
reject(new Error(stderr || `${toolName} exited with code ${code}`));
|
|
877
1064
|
}
|
|
878
1065
|
});
|
|
879
1066
|
});
|
|
880
1067
|
}
|
|
881
|
-
|
|
1068
|
+
/**
|
|
1069
|
+
* Resolve mysqldump, preferring the version the dump is headed for.
|
|
1070
|
+
*
|
|
1071
|
+
* `configManager.getBinaryPath('mysqldump')` keeps ONE path per tool name
|
|
1072
|
+
* with no version dimension, so on a machine with several MySQL versions
|
|
1073
|
+
* installed it returns whichever was registered first and still exists: a
|
|
1074
|
+
* 9.7.2 container was dumping through `mysql-9.6.0/bin/mysqldump`. The
|
|
1075
|
+
* bundled cache is asked first, exactly as the local backup path already
|
|
1076
|
+
* does, and the globally registered path stays the last resort.
|
|
1077
|
+
*/
|
|
1078
|
+
async getDumpPath(preferVersion) {
|
|
1079
|
+
const bundled = resolveBundledMysqlFamilyBinary({
|
|
1080
|
+
engine: 'mysql',
|
|
1081
|
+
tool: 'mysqldump',
|
|
1082
|
+
preferVersion,
|
|
1083
|
+
});
|
|
1084
|
+
if (bundled)
|
|
1085
|
+
return bundled.path;
|
|
882
1086
|
const configPath = await configManager.getBinaryPath('mysqldump');
|
|
883
1087
|
if (configPath)
|
|
884
1088
|
return configPath;
|