qrpa 1.1.33__py3-none-any.whl → 1.1.34__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of qrpa might be problematic. Click here for more details.

qrpa/db_migrator.py CHANGED
@@ -1,601 +1,601 @@
1
- #!/usr/bin/env python3
2
- # -*- coding: utf-8 -*-
3
- """
4
- 数据库迁移模块
5
-
6
- 这个模块提供了一个完整的数据库迁移解决方案,支持将本地Docker MySQL数据库的指定表
7
- 同步到远程服务器。
8
-
9
- 功能特性:
10
- - 自动导出本地数据库表
11
- - 通过SSH上传到远程服务器
12
- - 自动导入到远程数据库
13
- - 支持静默执行和交互式确认
14
- - 完整的错误处理和日志记录
15
-
16
- 作者: qsir
17
- 版本: 1.0.0
18
- """
19
-
20
- import os
21
- import sys
22
- import subprocess
23
- import tempfile
24
- import shutil
25
- import logging
26
- import platform
27
- import locale
28
- from typing import List, Dict, Optional, Tuple
29
- from dataclasses import dataclass
30
- from pathlib import Path
31
-
32
-
33
- @dataclass
34
- class DatabaseConfig:
35
- """数据库配置类"""
36
- host: str = "localhost"
37
- port: int = 3306
38
- user: str = "root"
39
- password: str = "123wyk"
40
- database: str = "lz"
41
- docker_container: str = "mysql"
42
-
43
-
44
- @dataclass
45
- class RemoteConfig:
46
- """远程服务器配置类"""
47
- ssh_host: str = "git@e3"
48
- temp_dir: str = "/tmp/db_migration"
49
- database: DatabaseConfig = None
50
-
51
- def __post_init__(self):
52
- if self.database is None:
53
- self.database = DatabaseConfig(docker_container="mysql")
54
-
55
-
56
- class DatabaseMigrator:
57
- """数据库迁移器主类"""
58
-
59
- def __init__(self,
60
- local_db: DatabaseConfig,
61
- remote_config: RemoteConfig,
62
- tables: List[str],
63
- silent: bool = False,
64
- log_level: str = "INFO"):
65
- """
66
- 初始化数据库迁移器
67
-
68
- Args:
69
- local_db: 本地数据库配置
70
- remote_config: 远程服务器配置
71
- tables: 要迁移的表列表
72
- silent: 静默模式(True=自动执行,False=需要确认)
73
- log_level: 日志级别
74
- """
75
- self.local_db = local_db
76
- self.remote_config = remote_config
77
- self.tables = tables
78
- self.silent = silent
79
-
80
- # 设置日志
81
- self.logger = self._setup_logger(log_level)
82
-
83
- # 临时文件目录
84
- self.temp_dir = None
85
-
86
- def _setup_logger(self, level: str) -> logging.Logger:
87
- """设置日志器"""
88
- logger = logging.getLogger("DatabaseMigrator")
89
- logger.setLevel(getattr(logging, level.upper()))
90
-
91
- if not logger.handlers:
92
- handler = logging.StreamHandler(sys.stdout)
93
- formatter = logging.Formatter(
94
- '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
95
- )
96
- handler.setFormatter(formatter)
97
- logger.addHandler(handler)
98
-
99
- return logger
100
-
101
- def _run_command(self, command: List[str], cwd: Optional[str] = None, encoding: str = "utf-8") -> Tuple[bool, str, str]:
102
- """
103
- 执行系统命令
104
-
105
- Args:
106
- command: 命令列表
107
- cwd: 工作目录
108
- encoding: 文本编码,默认utf-8
109
-
110
- Returns:
111
- (成功标志, 标准输出, 错误输出)
112
- """
113
- try:
114
- self.logger.debug(f"执行命令: {' '.join(command)}")
115
-
116
- # 在Windows系统上,使用二进制模式捕获输出以避免编码问题
117
- result = subprocess.run(
118
- command,
119
- cwd=cwd,
120
- capture_output=True,
121
- timeout=300 # 5分钟超时
122
- )
123
-
124
- success = result.returncode == 0
125
-
126
- # 手动解码输出,处理编码错误
127
- try:
128
- stdout = result.stdout.decode(encoding, errors='replace')
129
- stderr = result.stderr.decode(encoding, errors='replace')
130
- except AttributeError:
131
- # 如果已经是字符串,直接使用
132
- stdout = result.stdout or ""
133
- stderr = result.stderr or ""
134
-
135
- return success, stdout, stderr
136
-
137
- except subprocess.TimeoutExpired:
138
- self.logger.error("命令执行超时")
139
- return False, "", "命令执行超时"
140
- except Exception as e:
141
- self.logger.error(f"命令执行失败: {e}")
142
- return False, "", str(e)
143
-
144
- def _confirm_action(self, message: str) -> bool:
145
- """
146
- 确认操作
147
-
148
- Args:
149
- message: 确认消息
150
-
151
- Returns:
152
- 是否确认
153
- """
154
- if self.silent:
155
- self.logger.info(f"静默模式: {message} - 自动确认")
156
- return True
157
-
158
- while True:
159
- response = input(f"{message} (y/N): ").strip().lower()
160
- if response in ['y', 'yes']:
161
- return True
162
- elif response in ['n', 'no', '']:
163
- return False
164
- else:
165
- print("请输入 y/yes 或 n/no")
166
-
167
- def _create_temp_directory(self) -> str:
168
- """创建临时目录"""
169
- self.temp_dir = tempfile.mkdtemp(prefix="db_migration_")
170
- self.logger.info(f"创建临时目录: {self.temp_dir}")
171
- return self.temp_dir
172
-
173
- def _cleanup_temp_directory(self):
174
- """清理临时目录"""
175
- if self.temp_dir and os.path.exists(self.temp_dir):
176
- shutil.rmtree(self.temp_dir)
177
- self.logger.info(f"清理临时目录: {self.temp_dir}")
178
-
179
- def _detect_system_encoding(self) -> str:
180
- """检测系统编码"""
181
- # 在Windows上使用UTF-8,在其他系统上使用系统默认编码
182
- if platform.system() == "Windows":
183
- return "utf-8"
184
- else:
185
- return locale.getpreferredencoding() or "utf-8"
186
-
187
- def _run_mysql_command(self, command: List[str]) -> Tuple[bool, str, str]:
188
- """
189
- 专门用于运行MySQL命令的方法,处理编码问题
190
-
191
- Args:
192
- command: MySQL命令列表
193
-
194
- Returns:
195
- (成功标志, 标准输出, 错误输出)
196
- """
197
- try:
198
- self.logger.debug(f"执行MySQL命令: {' '.join(command)}")
199
-
200
- # 设置环境变量以确保UTF-8编码
201
- env = os.environ.copy()
202
- env['PYTHONIOENCODING'] = 'utf-8'
203
-
204
- # 运行命令
205
- result = subprocess.run(
206
- command,
207
- capture_output=True,
208
- timeout=300,
209
- env=env
210
- )
211
-
212
- success = result.returncode == 0
213
-
214
- # 尝试多种编码方式解码输出
215
- encodings = ['utf-8', 'utf-8-sig', 'latin1', 'cp1252']
216
- stdout = ""
217
- stderr = ""
218
-
219
- for encoding in encodings:
220
- try:
221
- stdout = result.stdout.decode(encoding)
222
- stderr = result.stderr.decode(encoding)
223
- break
224
- except UnicodeDecodeError:
225
- continue
226
-
227
- # 如果所有编码都失败,使用错误替换模式
228
- if not stdout and result.stdout:
229
- stdout = result.stdout.decode('utf-8', errors='replace')
230
- if not stderr and result.stderr:
231
- stderr = result.stderr.decode('utf-8', errors='replace')
232
-
233
- return success, stdout, stderr
234
-
235
- except subprocess.TimeoutExpired:
236
- self.logger.error("MySQL命令执行超时")
237
- return False, "", "MySQL命令执行超时"
238
- except Exception as e:
239
- self.logger.error(f"MySQL命令执行失败: {e}")
240
- return False, "", str(e)
241
-
242
- def _export_table(self, table: str) -> bool:
243
- """
244
- 导出单个表
245
-
246
- Args:
247
- table: 表名
248
-
249
- Returns:
250
- 是否成功
251
- """
252
- self.logger.info(f"正在导出表: {table}")
253
-
254
- output_file = os.path.join(self.temp_dir, f"{table}.sql")
255
-
256
- # 构建MySQL命令,在Windows上避免密码直接显示在命令行
257
- if platform.system() == "Windows":
258
- command = [
259
- "docker", "exec", "-e", f"MYSQL_PWD={self.local_db.password}",
260
- self.local_db.docker_container, "mysqldump",
261
- "-h", self.local_db.host,
262
- "-P", str(self.local_db.port),
263
- "-u", self.local_db.user,
264
- "--single-transaction",
265
- "--routines",
266
- "--triggers",
267
- "--set-gtid-purged=OFF",
268
- "--default-character-set=utf8mb4", # 确保使用UTF-8字符集
269
- "--skip-comments", # 跳过注释以减少编码问题
270
- self.local_db.database,
271
- table
272
- ]
273
- else:
274
- command = [
275
- "docker", "exec", self.local_db.docker_container, "mysqldump",
276
- "-h", self.local_db.host,
277
- "-P", str(self.local_db.port),
278
- "-u", self.local_db.user,
279
- f"-p{self.local_db.password}",
280
- "--single-transaction",
281
- "--routines",
282
- "--triggers",
283
- "--set-gtid-purged=OFF",
284
- "--default-character-set=utf8mb4", # 确保使用UTF-8字符集
285
- "--skip-comments", # 跳过注释以减少编码问题
286
- self.local_db.database,
287
- table
288
- ]
289
-
290
- # 使用专门的MySQL命令执行方法
291
- success, stdout, stderr = self._run_mysql_command(command)
292
-
293
- if success and stdout:
294
- # 将输出写入文件
295
- try:
296
- with open(output_file, 'w', encoding='utf-8') as f:
297
- f.write(stdout)
298
- self.logger.info(f"✓ 表 {table} 导出成功")
299
- return True
300
- except Exception as e:
301
- self.logger.error(f"✗ 表 {table} 写入文件失败: {e}")
302
- return False
303
- else:
304
- self.logger.error(f"✗ 表 {table} 导出失败: {stderr}")
305
- return False
306
-
307
- def _export_all_tables(self) -> bool:
308
- """导出所有表"""
309
- self.logger.info("开始导出本地数据库表...")
310
-
311
- for table in self.tables:
312
- if not self._export_table(table):
313
- return False
314
-
315
- self.logger.info("所有表导出完成!")
316
- return True
317
-
318
- def _upload_files_to_remote(self) -> bool:
319
- """上传文件到远程服务器"""
320
- self.logger.info("开始上传SQL文件到远程服务器...")
321
-
322
- # 创建远程目录
323
- command = ["ssh", self.remote_config.ssh_host, f"mkdir -p {self.remote_config.temp_dir}"]
324
- success, _, stderr = self._run_command(command)
325
-
326
- if not success:
327
- self.logger.error(f"无法创建远程目录: {stderr}")
328
- return False
329
-
330
- # 上传所有SQL文件
331
- for table in self.tables:
332
- local_file = os.path.join(self.temp_dir, f"{table}.sql")
333
- remote_path = f"{self.remote_config.ssh_host}:{self.remote_config.temp_dir}/"
334
-
335
- self.logger.info(f"正在上传 {table}.sql...")
336
-
337
- command = ["scp", local_file, remote_path]
338
- success, _, stderr = self._run_command(command)
339
-
340
- if success:
341
- self.logger.info(f"✓ {table}.sql 上传成功")
342
- else:
343
- self.logger.error(f"✗ {table}.sql 上传失败: {stderr}")
344
- return False
345
-
346
- self.logger.info("所有SQL文件上传完成!")
347
- return True
348
-
349
- def _write_script_file(self, content: str, file_path: str) -> bool:
350
- """
351
- 安全地写入脚本文件,确保跨平台兼容性
352
-
353
- Args:
354
- content: 脚本内容
355
- file_path: 文件路径
356
-
357
- Returns:
358
- 是否成功
359
- """
360
- try:
361
- # 确保使用Unix换行符
362
- content = content.replace('\r\n', '\n').replace('\r', '\n')
363
-
364
- # 使用二进制模式写入以避免换行符问题
365
- with open(file_path, 'wb') as f:
366
- f.write(content.encode('utf-8'))
367
-
368
- return True
369
- except Exception as e:
370
- self.logger.error(f"写入脚本文件失败: {e}")
371
- return False
372
-
373
- def _create_remote_import_script(self) -> str:
374
- """创建远程导入脚本"""
375
- script_lines = [
376
- "#!/bin/bash",
377
- "",
378
- "# 从参数获取配置",
379
- "REMOTE_DB_HOST=$1",
380
- "REMOTE_DB_PORT=$2",
381
- "REMOTE_DB_USER=$3",
382
- "REMOTE_DB_PASSWORD=$4",
383
- "REMOTE_DB_NAME=$5",
384
- "REMOTE_TEMP_DIR=$6",
385
- "REMOTE_DOCKER_CONTAINER=$7",
386
- "",
387
- 'echo "开始导入数据到远程数据库..."',
388
- "",
389
- "# 导入每个表",
390
- 'for sql_file in "$REMOTE_TEMP_DIR"/*.sql; do',
391
- ' if [ -f "$sql_file" ]; then',
392
- ' table_name=$(basename "$sql_file" .sql)',
393
- ' echo "正在导入表: $table_name"',
394
- " ",
395
- " # 先删除表中的数据(如果需要覆盖)",
396
- ' docker exec "$REMOTE_DOCKER_CONTAINER" mysql \\',
397
- ' -h "$REMOTE_DB_HOST" \\',
398
- ' -P "$REMOTE_DB_PORT" \\',
399
- ' -u "$REMOTE_DB_USER" \\',
400
- ' -p"$REMOTE_DB_PASSWORD" \\',
401
- ' "$REMOTE_DB_NAME" \\',
402
- ' -e "SET FOREIGN_KEY_CHECKS=0; DROP TABLE IF EXISTS $table_name; SET FOREIGN_KEY_CHECKS=1;"',
403
- " ",
404
- " # 导入SQL文件",
405
- ' docker exec -i "$REMOTE_DOCKER_CONTAINER" mysql \\',
406
- ' -h "$REMOTE_DB_HOST" \\',
407
- ' -P "$REMOTE_DB_PORT" \\',
408
- ' -u "$REMOTE_DB_USER" \\',
409
- ' -p"$REMOTE_DB_PASSWORD" \\',
410
- ' "$REMOTE_DB_NAME" < "$sql_file"',
411
- " ",
412
- " if [ $? -eq 0 ]; then",
413
- ' echo "✓ 表 $table_name 导入成功"',
414
- " else",
415
- ' echo "✗ 表 $table_name 导入失败"',
416
- " exit 1",
417
- " fi",
418
- " fi",
419
- "done",
420
- "",
421
- 'echo "所有表导入完成!"',
422
- "",
423
- "# 清理临时文件",
424
- 'rm -rf "$REMOTE_TEMP_DIR"',
425
- 'echo "临时文件已清理"'
426
- ]
427
-
428
- script_content = '\n'.join(script_lines)
429
- script_path = os.path.join(self.temp_dir, "remote_import.sh")
430
-
431
- if self._write_script_file(script_content, script_path):
432
- return script_path
433
- else:
434
- raise Exception("创建远程导入脚本失败")
435
-
436
- def _import_to_remote_database(self) -> bool:
437
- """导入数据到远程数据库"""
438
- self.logger.info("开始在远程服务器导入数据...")
439
-
440
- if not self.silent:
441
- self.logger.warning("注意: 这将覆盖远程服务器上的同名表!")
442
- if not self._confirm_action("确认要导入到远程数据库?"):
443
- self.logger.info("导入已跳过")
444
- return True
445
-
446
- # 创建远程导入脚本
447
- script_path = self._create_remote_import_script()
448
-
449
- # 上传脚本
450
- remote_script_path = f"{self.remote_config.ssh_host}:{self.remote_config.temp_dir}/remote_import.sh"
451
- command = ["scp", script_path, remote_script_path]
452
- success, _, stderr = self._run_command(command)
453
-
454
- if not success:
455
- self.logger.error(f"上传导入脚本失败: {stderr}")
456
- return False
457
-
458
- # 在远程服务器执行导入脚本
459
- remote_script = f"{self.remote_config.temp_dir}/remote_import.sh"
460
- remote_cmd = (
461
- f"dos2unix {remote_script} 2>/dev/null || sed -i 's/\\r$//' {remote_script} 2>/dev/null; "
462
- f"chmod +x {remote_script} && "
463
- f"{remote_script} "
464
- f"'{self.remote_config.database.host}' "
465
- f"'{self.remote_config.database.port}' "
466
- f"'{self.remote_config.database.user}' "
467
- f"'{self.remote_config.database.password}' "
468
- f"'{self.remote_config.database.database}' "
469
- f"'{self.remote_config.temp_dir}' "
470
- f"'{self.remote_config.database.docker_container}'"
471
- )
472
-
473
- command = ["ssh", self.remote_config.ssh_host, remote_cmd]
474
- success, stdout, stderr = self._run_command(command)
475
-
476
- if success:
477
- self.logger.info("✓ 远程数据库导入完成")
478
- if stdout:
479
- self.logger.info(f"导入输出: {stdout}")
480
- return True
481
- else:
482
- self.logger.error(f"✗ 远程数据库导入失败: {stderr}")
483
- return False
484
-
485
- def migrate(self) -> bool:
486
- """
487
- 执行完整的数据库迁移流程
488
-
489
- Returns:
490
- 是否成功
491
- """
492
- try:
493
- self.logger.info("==========================================")
494
- self.logger.info(" 数据库迁移工具 v1.0 (Python)")
495
- self.logger.info("==========================================")
496
- self.logger.info("本工具将从本地数据库导出指定表,并同步到远程服务器")
497
- self.logger.info("")
498
-
499
- self.logger.info("配置信息:")
500
- self.logger.info(f"本地数据库: {self.local_db.user}@{self.local_db.host}:{self.local_db.port}/{self.local_db.database}")
501
- self.logger.info(f"远程服务器: {self.remote_config.ssh_host}")
502
- self.logger.info(f"要迁移的表: {', '.join(self.tables)}")
503
- self.logger.info(f"执行模式: {'静默模式' if self.silent else '交互模式'}")
504
- self.logger.info("")
505
-
506
- if not self._confirm_action("是否继续执行迁移?"):
507
- self.logger.info("操作已取消")
508
- return False
509
-
510
- # 创建临时目录
511
- self._create_temp_directory()
512
-
513
- # 第1步: 导出本地数据
514
- self.logger.info("")
515
- self.logger.info("第1步: 导出本地数据库表")
516
- self.logger.info("----------------------------------------")
517
- if not self._export_all_tables():
518
- return False
519
-
520
- # 第2步: 上传到远程服务器
521
- self.logger.info("")
522
- self.logger.info("第2步: 上传文件到远程服务器")
523
- self.logger.info("----------------------------------------")
524
- if not self._upload_files_to_remote():
525
- return False
526
-
527
- # 第3步: 导入到远程数据库
528
- self.logger.info("")
529
- self.logger.info("第3步: 导入数据到远程数据库")
530
- self.logger.info("----------------------------------------")
531
- if not self._import_to_remote_database():
532
- return False
533
-
534
- self.logger.info("")
535
- self.logger.info("==========================================")
536
- self.logger.info(" 迁移完成!")
537
- self.logger.info("==========================================")
538
- self.logger.info("所有表已成功从本地数据库同步到远程服务器")
539
- self.logger.info(f"迁移的表: {', '.join(self.tables)}")
540
- self.logger.info("")
541
-
542
- return True
543
-
544
- except Exception as e:
545
- self.logger.error(f"迁移过程中发生错误: {e}")
546
- return False
547
- finally:
548
- # 清理临时文件
549
- self._cleanup_temp_directory()
550
-
551
-
552
- def create_default_migrator(silent: bool = False) -> DatabaseMigrator:
553
- """
554
- 创建默认配置的迁移器
555
-
556
- Args:
557
- silent: 是否静默执行
558
-
559
- Returns:
560
- DatabaseMigrator实例
561
- """
562
- # 本地数据库配置
563
- local_db = DatabaseConfig(
564
- host="localhost",
565
- port=3306,
566
- user="root",
567
- password="123wyk",
568
- database="lz",
569
- docker_container="mysql"
570
- )
571
-
572
- # 远程服务器配置
573
- remote_db = DatabaseConfig(
574
- host="localhost",
575
- port=3306,
576
- user="root",
577
- password="123wyk",
578
- database="lz",
579
- docker_container="mysql"
580
- )
581
-
582
- remote_config = RemoteConfig(
583
- ssh_host="git@ecslz",
584
- temp_dir="/tmp/db_migration",
585
- database=remote_db
586
- )
587
-
588
- # 要迁移的表
589
- tables = [
590
- "market_category",
591
- "market_country_sites",
592
- "market_product_ranking",
593
- "market_product_search_word"
594
- ]
595
-
596
- return DatabaseMigrator(
597
- local_db=local_db,
598
- remote_config=remote_config,
599
- tables=tables,
600
- silent=silent
1
+ #!/usr/bin/env python3
2
+ # -*- coding: utf-8 -*-
3
+ """
4
+ 数据库迁移模块
5
+
6
+ 这个模块提供了一个完整的数据库迁移解决方案,支持将本地Docker MySQL数据库的指定表
7
+ 同步到远程服务器。
8
+
9
+ 功能特性:
10
+ - 自动导出本地数据库表
11
+ - 通过SSH上传到远程服务器
12
+ - 自动导入到远程数据库
13
+ - 支持静默执行和交互式确认
14
+ - 完整的错误处理和日志记录
15
+
16
+ 作者: qsir
17
+ 版本: 1.0.0
18
+ """
19
+
20
+ import os
21
+ import sys
22
+ import subprocess
23
+ import tempfile
24
+ import shutil
25
+ import logging
26
+ import platform
27
+ import locale
28
+ from typing import List, Dict, Optional, Tuple
29
+ from dataclasses import dataclass
30
+ from pathlib import Path
31
+
32
+
33
+ @dataclass
34
+ class DatabaseConfig:
35
+ """数据库配置类"""
36
+ host: str = "localhost"
37
+ port: int = 3306
38
+ user: str = "root"
39
+ password: str = "123wyk"
40
+ database: str = "lz"
41
+ docker_container: str = "mysql"
42
+
43
+
44
+ @dataclass
45
+ class RemoteConfig:
46
+ """远程服务器配置类"""
47
+ ssh_host: str = "git@e3"
48
+ temp_dir: str = "/tmp/db_migration"
49
+ database: DatabaseConfig = None
50
+
51
+ def __post_init__(self):
52
+ if self.database is None:
53
+ self.database = DatabaseConfig(docker_container="mysql")
54
+
55
+
56
+ class DatabaseMigrator:
57
+ """数据库迁移器主类"""
58
+
59
+ def __init__(self,
60
+ local_db: DatabaseConfig,
61
+ remote_config: RemoteConfig,
62
+ tables: List[str],
63
+ silent: bool = False,
64
+ log_level: str = "INFO"):
65
+ """
66
+ 初始化数据库迁移器
67
+
68
+ Args:
69
+ local_db: 本地数据库配置
70
+ remote_config: 远程服务器配置
71
+ tables: 要迁移的表列表
72
+ silent: 静默模式(True=自动执行,False=需要确认)
73
+ log_level: 日志级别
74
+ """
75
+ self.local_db = local_db
76
+ self.remote_config = remote_config
77
+ self.tables = tables
78
+ self.silent = silent
79
+
80
+ # 设置日志
81
+ self.logger = self._setup_logger(log_level)
82
+
83
+ # 临时文件目录
84
+ self.temp_dir = None
85
+
86
+ def _setup_logger(self, level: str) -> logging.Logger:
87
+ """设置日志器"""
88
+ logger = logging.getLogger("DatabaseMigrator")
89
+ logger.setLevel(getattr(logging, level.upper()))
90
+
91
+ if not logger.handlers:
92
+ handler = logging.StreamHandler(sys.stdout)
93
+ formatter = logging.Formatter(
94
+ '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
95
+ )
96
+ handler.setFormatter(formatter)
97
+ logger.addHandler(handler)
98
+
99
+ return logger
100
+
101
+ def _run_command(self, command: List[str], cwd: Optional[str] = None, encoding: str = "utf-8") -> Tuple[bool, str, str]:
102
+ """
103
+ 执行系统命令
104
+
105
+ Args:
106
+ command: 命令列表
107
+ cwd: 工作目录
108
+ encoding: 文本编码,默认utf-8
109
+
110
+ Returns:
111
+ (成功标志, 标准输出, 错误输出)
112
+ """
113
+ try:
114
+ self.logger.debug(f"执行命令: {' '.join(command)}")
115
+
116
+ # 在Windows系统上,使用二进制模式捕获输出以避免编码问题
117
+ result = subprocess.run(
118
+ command,
119
+ cwd=cwd,
120
+ capture_output=True,
121
+ timeout=300 # 5分钟超时
122
+ )
123
+
124
+ success = result.returncode == 0
125
+
126
+ # 手动解码输出,处理编码错误
127
+ try:
128
+ stdout = result.stdout.decode(encoding, errors='replace')
129
+ stderr = result.stderr.decode(encoding, errors='replace')
130
+ except AttributeError:
131
+ # 如果已经是字符串,直接使用
132
+ stdout = result.stdout or ""
133
+ stderr = result.stderr or ""
134
+
135
+ return success, stdout, stderr
136
+
137
+ except subprocess.TimeoutExpired:
138
+ self.logger.error("命令执行超时")
139
+ return False, "", "命令执行超时"
140
+ except Exception as e:
141
+ self.logger.error(f"命令执行失败: {e}")
142
+ return False, "", str(e)
143
+
144
+ def _confirm_action(self, message: str) -> bool:
145
+ """
146
+ 确认操作
147
+
148
+ Args:
149
+ message: 确认消息
150
+
151
+ Returns:
152
+ 是否确认
153
+ """
154
+ if self.silent:
155
+ self.logger.info(f"静默模式: {message} - 自动确认")
156
+ return True
157
+
158
+ while True:
159
+ response = input(f"{message} (y/N): ").strip().lower()
160
+ if response in ['y', 'yes']:
161
+ return True
162
+ elif response in ['n', 'no', '']:
163
+ return False
164
+ else:
165
+ print("请输入 y/yes 或 n/no")
166
+
167
+ def _create_temp_directory(self) -> str:
168
+ """创建临时目录"""
169
+ self.temp_dir = tempfile.mkdtemp(prefix="db_migration_")
170
+ self.logger.info(f"创建临时目录: {self.temp_dir}")
171
+ return self.temp_dir
172
+
173
+ def _cleanup_temp_directory(self):
174
+ """清理临时目录"""
175
+ if self.temp_dir and os.path.exists(self.temp_dir):
176
+ shutil.rmtree(self.temp_dir)
177
+ self.logger.info(f"清理临时目录: {self.temp_dir}")
178
+
179
+ def _detect_system_encoding(self) -> str:
180
+ """检测系统编码"""
181
+ # 在Windows上使用UTF-8,在其他系统上使用系统默认编码
182
+ if platform.system() == "Windows":
183
+ return "utf-8"
184
+ else:
185
+ return locale.getpreferredencoding() or "utf-8"
186
+
187
+ def _run_mysql_command(self, command: List[str]) -> Tuple[bool, str, str]:
188
+ """
189
+ 专门用于运行MySQL命令的方法,处理编码问题
190
+
191
+ Args:
192
+ command: MySQL命令列表
193
+
194
+ Returns:
195
+ (成功标志, 标准输出, 错误输出)
196
+ """
197
+ try:
198
+ self.logger.debug(f"执行MySQL命令: {' '.join(command)}")
199
+
200
+ # 设置环境变量以确保UTF-8编码
201
+ env = os.environ.copy()
202
+ env['PYTHONIOENCODING'] = 'utf-8'
203
+
204
+ # 运行命令
205
+ result = subprocess.run(
206
+ command,
207
+ capture_output=True,
208
+ timeout=300,
209
+ env=env
210
+ )
211
+
212
+ success = result.returncode == 0
213
+
214
+ # 尝试多种编码方式解码输出
215
+ encodings = ['utf-8', 'utf-8-sig', 'latin1', 'cp1252']
216
+ stdout = ""
217
+ stderr = ""
218
+
219
+ for encoding in encodings:
220
+ try:
221
+ stdout = result.stdout.decode(encoding)
222
+ stderr = result.stderr.decode(encoding)
223
+ break
224
+ except UnicodeDecodeError:
225
+ continue
226
+
227
+ # 如果所有编码都失败,使用错误替换模式
228
+ if not stdout and result.stdout:
229
+ stdout = result.stdout.decode('utf-8', errors='replace')
230
+ if not stderr and result.stderr:
231
+ stderr = result.stderr.decode('utf-8', errors='replace')
232
+
233
+ return success, stdout, stderr
234
+
235
+ except subprocess.TimeoutExpired:
236
+ self.logger.error("MySQL命令执行超时")
237
+ return False, "", "MySQL命令执行超时"
238
+ except Exception as e:
239
+ self.logger.error(f"MySQL命令执行失败: {e}")
240
+ return False, "", str(e)
241
+
242
+ def _export_table(self, table: str) -> bool:
243
+ """
244
+ 导出单个表
245
+
246
+ Args:
247
+ table: 表名
248
+
249
+ Returns:
250
+ 是否成功
251
+ """
252
+ self.logger.info(f"正在导出表: {table}")
253
+
254
+ output_file = os.path.join(self.temp_dir, f"{table}.sql")
255
+
256
+ # 构建MySQL命令,在Windows上避免密码直接显示在命令行
257
+ if platform.system() == "Windows":
258
+ command = [
259
+ "docker", "exec", "-e", f"MYSQL_PWD={self.local_db.password}",
260
+ self.local_db.docker_container, "mysqldump",
261
+ "-h", self.local_db.host,
262
+ "-P", str(self.local_db.port),
263
+ "-u", self.local_db.user,
264
+ "--single-transaction",
265
+ "--routines",
266
+ "--triggers",
267
+ "--set-gtid-purged=OFF",
268
+ "--default-character-set=utf8mb4", # 确保使用UTF-8字符集
269
+ "--skip-comments", # 跳过注释以减少编码问题
270
+ self.local_db.database,
271
+ table
272
+ ]
273
+ else:
274
+ command = [
275
+ "docker", "exec", self.local_db.docker_container, "mysqldump",
276
+ "-h", self.local_db.host,
277
+ "-P", str(self.local_db.port),
278
+ "-u", self.local_db.user,
279
+ f"-p{self.local_db.password}",
280
+ "--single-transaction",
281
+ "--routines",
282
+ "--triggers",
283
+ "--set-gtid-purged=OFF",
284
+ "--default-character-set=utf8mb4", # 确保使用UTF-8字符集
285
+ "--skip-comments", # 跳过注释以减少编码问题
286
+ self.local_db.database,
287
+ table
288
+ ]
289
+
290
+ # 使用专门的MySQL命令执行方法
291
+ success, stdout, stderr = self._run_mysql_command(command)
292
+
293
+ if success and stdout:
294
+ # 将输出写入文件
295
+ try:
296
+ with open(output_file, 'w', encoding='utf-8') as f:
297
+ f.write(stdout)
298
+ self.logger.info(f"✓ 表 {table} 导出成功")
299
+ return True
300
+ except Exception as e:
301
+ self.logger.error(f"✗ 表 {table} 写入文件失败: {e}")
302
+ return False
303
+ else:
304
+ self.logger.error(f"✗ 表 {table} 导出失败: {stderr}")
305
+ return False
306
+
307
+ def _export_all_tables(self) -> bool:
308
+ """导出所有表"""
309
+ self.logger.info("开始导出本地数据库表...")
310
+
311
+ for table in self.tables:
312
+ if not self._export_table(table):
313
+ return False
314
+
315
+ self.logger.info("所有表导出完成!")
316
+ return True
317
+
318
+ def _upload_files_to_remote(self) -> bool:
319
+ """上传文件到远程服务器"""
320
+ self.logger.info("开始上传SQL文件到远程服务器...")
321
+
322
+ # 创建远程目录
323
+ command = ["ssh", self.remote_config.ssh_host, f"mkdir -p {self.remote_config.temp_dir}"]
324
+ success, _, stderr = self._run_command(command)
325
+
326
+ if not success:
327
+ self.logger.error(f"无法创建远程目录: {stderr}")
328
+ return False
329
+
330
+ # 上传所有SQL文件
331
+ for table in self.tables:
332
+ local_file = os.path.join(self.temp_dir, f"{table}.sql")
333
+ remote_path = f"{self.remote_config.ssh_host}:{self.remote_config.temp_dir}/"
334
+
335
+ self.logger.info(f"正在上传 {table}.sql...")
336
+
337
+ command = ["scp", local_file, remote_path]
338
+ success, _, stderr = self._run_command(command)
339
+
340
+ if success:
341
+ self.logger.info(f"✓ {table}.sql 上传成功")
342
+ else:
343
+ self.logger.error(f"✗ {table}.sql 上传失败: {stderr}")
344
+ return False
345
+
346
+ self.logger.info("所有SQL文件上传完成!")
347
+ return True
348
+
349
+ def _write_script_file(self, content: str, file_path: str) -> bool:
350
+ """
351
+ 安全地写入脚本文件,确保跨平台兼容性
352
+
353
+ Args:
354
+ content: 脚本内容
355
+ file_path: 文件路径
356
+
357
+ Returns:
358
+ 是否成功
359
+ """
360
+ try:
361
+ # 确保使用Unix换行符
362
+ content = content.replace('\r\n', '\n').replace('\r', '\n')
363
+
364
+ # 使用二进制模式写入以避免换行符问题
365
+ with open(file_path, 'wb') as f:
366
+ f.write(content.encode('utf-8'))
367
+
368
+ return True
369
+ except Exception as e:
370
+ self.logger.error(f"写入脚本文件失败: {e}")
371
+ return False
372
+
373
+ def _create_remote_import_script(self) -> str:
374
+ """创建远程导入脚本"""
375
+ script_lines = [
376
+ "#!/bin/bash",
377
+ "",
378
+ "# 从参数获取配置",
379
+ "REMOTE_DB_HOST=$1",
380
+ "REMOTE_DB_PORT=$2",
381
+ "REMOTE_DB_USER=$3",
382
+ "REMOTE_DB_PASSWORD=$4",
383
+ "REMOTE_DB_NAME=$5",
384
+ "REMOTE_TEMP_DIR=$6",
385
+ "REMOTE_DOCKER_CONTAINER=$7",
386
+ "",
387
+ 'echo "开始导入数据到远程数据库..."',
388
+ "",
389
+ "# 导入每个表",
390
+ 'for sql_file in "$REMOTE_TEMP_DIR"/*.sql; do',
391
+ ' if [ -f "$sql_file" ]; then',
392
+ ' table_name=$(basename "$sql_file" .sql)',
393
+ ' echo "正在导入表: $table_name"',
394
+ " ",
395
+ " # 先删除表中的数据(如果需要覆盖)",
396
+ ' docker exec "$REMOTE_DOCKER_CONTAINER" mysql \\',
397
+ ' -h "$REMOTE_DB_HOST" \\',
398
+ ' -P "$REMOTE_DB_PORT" \\',
399
+ ' -u "$REMOTE_DB_USER" \\',
400
+ ' -p"$REMOTE_DB_PASSWORD" \\',
401
+ ' "$REMOTE_DB_NAME" \\',
402
+ ' -e "SET FOREIGN_KEY_CHECKS=0; DROP TABLE IF EXISTS $table_name; SET FOREIGN_KEY_CHECKS=1;"',
403
+ " ",
404
+ " # 导入SQL文件",
405
+ ' docker exec -i "$REMOTE_DOCKER_CONTAINER" mysql \\',
406
+ ' -h "$REMOTE_DB_HOST" \\',
407
+ ' -P "$REMOTE_DB_PORT" \\',
408
+ ' -u "$REMOTE_DB_USER" \\',
409
+ ' -p"$REMOTE_DB_PASSWORD" \\',
410
+ ' "$REMOTE_DB_NAME" < "$sql_file"',
411
+ " ",
412
+ " if [ $? -eq 0 ]; then",
413
+ ' echo "✓ 表 $table_name 导入成功"',
414
+ " else",
415
+ ' echo "✗ 表 $table_name 导入失败"',
416
+ " exit 1",
417
+ " fi",
418
+ " fi",
419
+ "done",
420
+ "",
421
+ 'echo "所有表导入完成!"',
422
+ "",
423
+ "# 清理临时文件",
424
+ 'rm -rf "$REMOTE_TEMP_DIR"',
425
+ 'echo "临时文件已清理"'
426
+ ]
427
+
428
+ script_content = '\n'.join(script_lines)
429
+ script_path = os.path.join(self.temp_dir, "remote_import.sh")
430
+
431
+ if self._write_script_file(script_content, script_path):
432
+ return script_path
433
+ else:
434
+ raise Exception("创建远程导入脚本失败")
435
+
436
+ def _import_to_remote_database(self) -> bool:
437
+ """导入数据到远程数据库"""
438
+ self.logger.info("开始在远程服务器导入数据...")
439
+
440
+ if not self.silent:
441
+ self.logger.warning("注意: 这将覆盖远程服务器上的同名表!")
442
+ if not self._confirm_action("确认要导入到远程数据库?"):
443
+ self.logger.info("导入已跳过")
444
+ return True
445
+
446
+ # 创建远程导入脚本
447
+ script_path = self._create_remote_import_script()
448
+
449
+ # 上传脚本
450
+ remote_script_path = f"{self.remote_config.ssh_host}:{self.remote_config.temp_dir}/remote_import.sh"
451
+ command = ["scp", script_path, remote_script_path]
452
+ success, _, stderr = self._run_command(command)
453
+
454
+ if not success:
455
+ self.logger.error(f"上传导入脚本失败: {stderr}")
456
+ return False
457
+
458
+ # 在远程服务器执行导入脚本
459
+ remote_script = f"{self.remote_config.temp_dir}/remote_import.sh"
460
+ remote_cmd = (
461
+ f"dos2unix {remote_script} 2>/dev/null || sed -i 's/\\r$//' {remote_script} 2>/dev/null; "
462
+ f"chmod +x {remote_script} && "
463
+ f"{remote_script} "
464
+ f"'{self.remote_config.database.host}' "
465
+ f"'{self.remote_config.database.port}' "
466
+ f"'{self.remote_config.database.user}' "
467
+ f"'{self.remote_config.database.password}' "
468
+ f"'{self.remote_config.database.database}' "
469
+ f"'{self.remote_config.temp_dir}' "
470
+ f"'{self.remote_config.database.docker_container}'"
471
+ )
472
+
473
+ command = ["ssh", self.remote_config.ssh_host, remote_cmd]
474
+ success, stdout, stderr = self._run_command(command)
475
+
476
+ if success:
477
+ self.logger.info("✓ 远程数据库导入完成")
478
+ if stdout:
479
+ self.logger.info(f"导入输出: {stdout}")
480
+ return True
481
+ else:
482
+ self.logger.error(f"✗ 远程数据库导入失败: {stderr}")
483
+ return False
484
+
485
+ def migrate(self) -> bool:
486
+ """
487
+ 执行完整的数据库迁移流程
488
+
489
+ Returns:
490
+ 是否成功
491
+ """
492
+ try:
493
+ self.logger.info("==========================================")
494
+ self.logger.info(" 数据库迁移工具 v1.0 (Python)")
495
+ self.logger.info("==========================================")
496
+ self.logger.info("本工具将从本地数据库导出指定表,并同步到远程服务器")
497
+ self.logger.info("")
498
+
499
+ self.logger.info("配置信息:")
500
+ self.logger.info(f"本地数据库: {self.local_db.user}@{self.local_db.host}:{self.local_db.port}/{self.local_db.database}")
501
+ self.logger.info(f"远程服务器: {self.remote_config.ssh_host}")
502
+ self.logger.info(f"要迁移的表: {', '.join(self.tables)}")
503
+ self.logger.info(f"执行模式: {'静默模式' if self.silent else '交互模式'}")
504
+ self.logger.info("")
505
+
506
+ if not self._confirm_action("是否继续执行迁移?"):
507
+ self.logger.info("操作已取消")
508
+ return False
509
+
510
+ # 创建临时目录
511
+ self._create_temp_directory()
512
+
513
+ # 第1步: 导出本地数据
514
+ self.logger.info("")
515
+ self.logger.info("第1步: 导出本地数据库表")
516
+ self.logger.info("----------------------------------------")
517
+ if not self._export_all_tables():
518
+ return False
519
+
520
+ # 第2步: 上传到远程服务器
521
+ self.logger.info("")
522
+ self.logger.info("第2步: 上传文件到远程服务器")
523
+ self.logger.info("----------------------------------------")
524
+ if not self._upload_files_to_remote():
525
+ return False
526
+
527
+ # 第3步: 导入到远程数据库
528
+ self.logger.info("")
529
+ self.logger.info("第3步: 导入数据到远程数据库")
530
+ self.logger.info("----------------------------------------")
531
+ if not self._import_to_remote_database():
532
+ return False
533
+
534
+ self.logger.info("")
535
+ self.logger.info("==========================================")
536
+ self.logger.info(" 迁移完成!")
537
+ self.logger.info("==========================================")
538
+ self.logger.info("所有表已成功从本地数据库同步到远程服务器")
539
+ self.logger.info(f"迁移的表: {', '.join(self.tables)}")
540
+ self.logger.info("")
541
+
542
+ return True
543
+
544
+ except Exception as e:
545
+ self.logger.error(f"迁移过程中发生错误: {e}")
546
+ return False
547
+ finally:
548
+ # 清理临时文件
549
+ self._cleanup_temp_directory()
550
+
551
+
552
+ def create_default_migrator(silent: bool = False) -> DatabaseMigrator:
553
+ """
554
+ 创建默认配置的迁移器
555
+
556
+ Args:
557
+ silent: 是否静默执行
558
+
559
+ Returns:
560
+ DatabaseMigrator实例
561
+ """
562
+ # 本地数据库配置
563
+ local_db = DatabaseConfig(
564
+ host="localhost",
565
+ port=3306,
566
+ user="root",
567
+ password="123wyk",
568
+ database="lz",
569
+ docker_container="mysql"
570
+ )
571
+
572
+ # 远程服务器配置
573
+ remote_db = DatabaseConfig(
574
+ host="localhost",
575
+ port=3306,
576
+ user="root",
577
+ password="123wyk",
578
+ database="lz",
579
+ docker_container="mysql"
580
+ )
581
+
582
+ remote_config = RemoteConfig(
583
+ ssh_host="git@ecslz",
584
+ temp_dir="/tmp/db_migration",
585
+ database=remote_db
586
+ )
587
+
588
+ # 要迁移的表
589
+ tables = [
590
+ "market_category",
591
+ "market_country_sites",
592
+ "market_product_ranking",
593
+ "market_product_search_word"
594
+ ]
595
+
596
+ return DatabaseMigrator(
597
+ local_db=local_db,
598
+ remote_config=remote_config,
599
+ tables=tables,
600
+ silent=silent
601
601
  )