pyxllib 0.3.197__py3-none-any.whl → 3.201.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyxllib/__init__.py +14 -21
- pyxllib/algo/__init__.py +8 -8
- pyxllib/algo/disjoint.py +54 -54
- pyxllib/algo/geo.py +537 -541
- pyxllib/algo/intervals.py +964 -964
- pyxllib/algo/matcher.py +389 -389
- pyxllib/algo/newbie.py +166 -166
- pyxllib/algo/pupil.py +629 -629
- pyxllib/algo/shapelylib.py +67 -67
- pyxllib/algo/specialist.py +241 -241
- pyxllib/algo/stat.py +494 -494
- pyxllib/algo/treelib.py +145 -149
- pyxllib/algo/unitlib.py +62 -66
- pyxllib/autogui/__init__.py +5 -5
- pyxllib/autogui/activewin.py +246 -246
- pyxllib/autogui/all.py +9 -9
- pyxllib/autogui/autogui.py +846 -852
- pyxllib/autogui/uiautolib.py +362 -362
- pyxllib/autogui/virtualkey.py +102 -102
- pyxllib/autogui/wechat.py +827 -827
- pyxllib/autogui/wechat_msg.py +421 -421
- pyxllib/autogui/wxautolib.py +84 -84
- pyxllib/cv/__init__.py +5 -5
- pyxllib/cv/expert.py +267 -267
- pyxllib/cv/imfile.py +159 -159
- pyxllib/cv/imhash.py +39 -39
- pyxllib/cv/pupil.py +9 -9
- pyxllib/cv/rgbfmt.py +1525 -1525
- pyxllib/cv/slidercaptcha.py +137 -137
- pyxllib/cv/trackbartools.py +251 -251
- pyxllib/cv/xlcvlib.py +1040 -1040
- pyxllib/cv/xlpillib.py +423 -423
- pyxllib/data/echarts.py +236 -240
- pyxllib/data/jsonlib.py +85 -89
- pyxllib/data/oss.py +72 -72
- pyxllib/data/pglib.py +1111 -1127
- pyxllib/data/sqlite.py +568 -568
- pyxllib/data/sqllib.py +297 -297
- pyxllib/ext/JLineViewer.py +505 -505
- pyxllib/ext/__init__.py +6 -6
- pyxllib/ext/demolib.py +251 -246
- pyxllib/ext/drissionlib.py +277 -277
- pyxllib/ext/kq5034lib.py +12 -12
- pyxllib/ext/qt.py +449 -449
- pyxllib/ext/robustprocfile.py +493 -497
- pyxllib/ext/seleniumlib.py +76 -76
- pyxllib/ext/tk.py +173 -173
- pyxllib/ext/unixlib.py +821 -827
- pyxllib/ext/utools.py +345 -351
- pyxllib/ext/webhook.py +124 -119
- pyxllib/ext/win32lib.py +40 -40
- pyxllib/ext/wjxlib.py +91 -88
- pyxllib/ext/wpsapi.py +124 -124
- pyxllib/ext/xlwork.py +9 -9
- pyxllib/ext/yuquelib.py +1110 -1105
- pyxllib/file/__init__.py +17 -17
- pyxllib/file/docxlib.py +757 -761
- pyxllib/file/gitlib.py +309 -309
- pyxllib/file/libreoffice.py +165 -165
- pyxllib/file/movielib.py +144 -148
- pyxllib/file/newbie.py +10 -10
- pyxllib/file/onenotelib.py +1469 -1469
- pyxllib/file/packlib/__init__.py +330 -330
- pyxllib/file/packlib/zipfile.py +2441 -2441
- pyxllib/file/pdflib.py +422 -426
- pyxllib/file/pupil.py +185 -185
- pyxllib/file/specialist/__init__.py +681 -685
- pyxllib/file/specialist/dirlib.py +799 -799
- pyxllib/file/specialist/download.py +193 -193
- pyxllib/file/specialist/filelib.py +2825 -2829
- pyxllib/file/xlsxlib.py +3122 -3131
- pyxllib/file/xlsyncfile.py +341 -341
- pyxllib/prog/__init__.py +5 -5
- pyxllib/prog/cachetools.py +58 -64
- pyxllib/prog/deprecatedlib.py +233 -233
- pyxllib/prog/filelock.py +42 -42
- pyxllib/prog/ipyexec.py +253 -253
- pyxllib/prog/multiprogs.py +940 -940
- pyxllib/prog/newbie.py +451 -451
- pyxllib/prog/pupil.py +1208 -1197
- pyxllib/prog/sitepackages.py +33 -33
- pyxllib/prog/specialist/__init__.py +348 -391
- pyxllib/prog/specialist/bc.py +203 -203
- pyxllib/prog/specialist/browser.py +497 -497
- pyxllib/prog/specialist/common.py +347 -347
- pyxllib/prog/specialist/datetime.py +198 -198
- pyxllib/prog/specialist/tictoc.py +240 -240
- pyxllib/prog/specialist/xllog.py +180 -180
- pyxllib/prog/xlosenv.py +110 -108
- pyxllib/stdlib/__init__.py +17 -17
- pyxllib/stdlib/tablepyxl/__init__.py +10 -10
- pyxllib/stdlib/tablepyxl/style.py +303 -303
- pyxllib/stdlib/tablepyxl/tablepyxl.py +130 -130
- pyxllib/text/__init__.py +8 -8
- pyxllib/text/ahocorasick.py +36 -39
- pyxllib/text/airscript.js +754 -744
- pyxllib/text/charclasslib.py +121 -121
- pyxllib/text/jiebalib.py +267 -267
- pyxllib/text/jinjalib.py +27 -32
- pyxllib/text/jsa_ai_prompt.md +271 -271
- pyxllib/text/jscode.py +922 -922
- pyxllib/text/latex/__init__.py +158 -158
- pyxllib/text/levenshtein.py +303 -303
- pyxllib/text/nestenv.py +1215 -1215
- pyxllib/text/newbie.py +300 -300
- pyxllib/text/pupil/__init__.py +8 -8
- pyxllib/text/pupil/common.py +1121 -1121
- pyxllib/text/pupil/xlalign.py +326 -326
- pyxllib/text/pycode.py +47 -47
- pyxllib/text/specialist/__init__.py +8 -8
- pyxllib/text/specialist/common.py +112 -112
- pyxllib/text/specialist/ptag.py +186 -186
- pyxllib/text/spellchecker.py +172 -172
- pyxllib/text/templates/echart_base.html +10 -10
- pyxllib/text/templates/highlight_code.html +16 -16
- pyxllib/text/templates/latex_editor.html +102 -102
- pyxllib/text/vbacode.py +17 -17
- pyxllib/text/xmllib.py +741 -747
- pyxllib/xl.py +42 -39
- pyxllib/xlcv.py +17 -17
- pyxllib-3.201.1.dist-info/METADATA +296 -0
- pyxllib-3.201.1.dist-info/RECORD +125 -0
- {pyxllib-0.3.197.dist-info → pyxllib-3.201.1.dist-info}/licenses/LICENSE +190 -190
- pyxllib/ext/old.py +0 -663
- pyxllib-0.3.197.dist-info/METADATA +0 -48
- pyxllib-0.3.197.dist-info/RECORD +0 -126
- {pyxllib-0.3.197.dist-info → pyxllib-3.201.1.dist-info}/WHEEL +0 -0
pyxllib/file/xlsyncfile.py
CHANGED
@@ -1,341 +1,341 @@
|
|
1
|
-
#!/usr/bin/env python3
|
2
|
-
# -*- coding: utf-8 -*-
|
3
|
-
# @Author : 陈坤泽
|
4
|
-
# @Email : 877362867@qq.com
|
5
|
-
# @Date : 2024/10/12
|
6
|
-
|
7
|
-
"""
|
8
|
-
我们自己的一套利用后端服务进行文件同步的工具
|
9
|
-
"""
|
10
|
-
|
11
|
-
import os
|
12
|
-
import time
|
13
|
-
|
14
|
-
from loguru import logger
|
15
|
-
from tqdm import tqdm
|
16
|
-
import requests
|
17
|
-
|
18
|
-
from pyxllib.file.specialist import XlPath, GetEtag
|
19
|
-
from pyxllib.prog.pupil import format_exception
|
20
|
-
|
21
|
-
|
22
|
-
class SyncFileClient:
|
23
|
-
"""
|
24
|
-
|
25
|
-
注意:该类填写的路径,一律是相对local_root、remote_root的路径,不是当前实际工作目录下的路径!
|
26
|
-
可以通过把local_host设为空的来实现一些特殊精细化的操作
|
27
|
-
但一般推荐还是都把local_root、remote_root设置上
|
28
|
-
"""
|
29
|
-
|
30
|
-
def __1_basic(self):
|
31
|
-
pass
|
32
|
-
|
33
|
-
def __init__(self, host, token=None):
|
34
|
-
"""
|
35
|
-
:param host: 可以只写主机名
|
36
|
-
"""
|
37
|
-
token = token or os.getenv("XL_COMMON_SERVER_TOKEN")
|
38
|
-
self.headers = {
|
39
|
-
'Authorization': f'Bearer {token}',
|
40
|
-
}
|
41
|
-
self.local_root = None
|
42
|
-
self.remote_root = None
|
43
|
-
self.host = self.link_host(host)
|
44
|
-
|
45
|
-
def link_host(self, hostname):
|
46
|
-
from requests.exceptions import Timeout
|
47
|
-
|
48
|
-
host = f'{os.getenv("MAIN_WEBSITE")}/{hostname}'
|
49
|
-
resp = requests.get(f'{host}/common/get_local_server', headers=self.headers)
|
50
|
-
ip = resp.json()['ip']
|
51
|
-
|
52
|
-
if ip:
|
53
|
-
try:
|
54
|
-
resp = requests.get(f'{ip}/healthz', timeout=5)
|
55
|
-
if resp.status_code == 200 and resp.json()['status'] == 'ok':
|
56
|
-
host = ip
|
57
|
-
except Timeout:
|
58
|
-
pass
|
59
|
-
except Exception as e:
|
60
|
-
raise e
|
61
|
-
|
62
|
-
remote_root = requests.get(f'{host}/common/get_wkdir', headers=self.headers).json()['wkdir']
|
63
|
-
self.remote_root = XlPath(remote_root)
|
64
|
-
|
65
|
-
return host
|
66
|
-
|
67
|
-
def get_abs_local_path(self, local_path, remote_path=None):
|
68
|
-
# 1 如果需要,从远程路径推测本地路径
|
69
|
-
if local_path is None and remote_path is not None:
|
70
|
-
local_path = remote_path
|
71
|
-
|
72
|
-
# 2 本地存储的文件路径
|
73
|
-
if self.local_root:
|
74
|
-
return self.local_root / local_path
|
75
|
-
else:
|
76
|
-
return XlPath(local_path)
|
77
|
-
|
78
|
-
def get_abs_remote_path(self, remote_path, local_path=None):
|
79
|
-
# 1 如果需要,从本地路径推测远程路径
|
80
|
-
if remote_path is None and local_path is not None:
|
81
|
-
remote_path = local_path
|
82
|
-
|
83
|
-
# 2 远程存储的文件路径
|
84
|
-
if self.remote_root:
|
85
|
-
return self.remote_root / remote_path
|
86
|
-
else:
|
87
|
-
return XlPath(remote_path)
|
88
|
-
|
89
|
-
def __2_sync(self):
|
90
|
-
pass
|
91
|
-
|
92
|
-
def upload_file(self, local_file=None, remote_file=None, relpath=True):
|
93
|
-
"""
|
94
|
-
:param str remote_file: 要存储的远程文件路径,默认可以不指定,自动通过local_file存储到对称位置
|
95
|
-
|
96
|
-
使用示例:
|
97
|
-
sfc = SyncFileClient('codepc_mi15')
|
98
|
-
sfc.upload_file('abc/README.md')
|
99
|
-
sfc.upload_file('abc/README.md', 'bcd/1.md')
|
100
|
-
"""
|
101
|
-
file = [local_file]
|
102
|
-
if remote_file:
|
103
|
-
file.append(remote_file)
|
104
|
-
return self.upload_files([file], relpath=relpath)
|
105
|
-
|
106
|
-
def upload_files(self, files, relpath=True):
|
107
|
-
"""
|
108
|
-
:param list[local_file<, remote_path>] files:
|
109
|
-
remote_path可选,未写的时候会自动填充
|
110
|
-
:return:
|
111
|
-
|
112
|
-
todo 加进度条?不过问了下gpt,这个实现好像有点复杂,先不折腾了。本来其实就推荐单个单个文件上传,在外部进行进度展示管控。
|
113
|
-
|
114
|
-
使用示例:
|
115
|
-
sfc = SyncFileClient('codepc_mi15')
|
116
|
-
print(sfc.upload_files(['data/d231120禅宗1期4阶.xlsx', 'data/d231120禅宗1期4阶2.xlsx']))
|
117
|
-
"""
|
118
|
-
# 1 映射出目标路径
|
119
|
-
files2 = []
|
120
|
-
for file in files:
|
121
|
-
if not isinstance(file, (list, tuple)):
|
122
|
-
local_file, remote_file = file, None
|
123
|
-
else:
|
124
|
-
local_file, remote_file = file
|
125
|
-
|
126
|
-
if relpath:
|
127
|
-
local_file = self.get_abs_local_path(local_file, remote_file)
|
128
|
-
remote_file = self.get_abs_remote_path(remote_file, local_file)
|
129
|
-
|
130
|
-
files2.append([local_file, remote_file.as_posix()])
|
131
|
-
|
132
|
-
files3 = [('files', (remote_file, open(local_file, 'rb'))) for local_file, remote_file in files2]
|
133
|
-
|
134
|
-
# 4 发送请求
|
135
|
-
resp = requests.post(f'{self.host}/common/upload_files',
|
136
|
-
headers=self.headers,
|
137
|
-
files=files3)
|
138
|
-
return resp.json()
|
139
|
-
|
140
|
-
def download_file(self, remote_file=None, local_file=None, relpath=True):
|
141
|
-
"""
|
142
|
-
:param str remote_file: 要下载的远程文件路径,相对get_wkdir下的路径
|
143
|
-
:param str local_file: 下载到本地的文件路径,默认可以不指定,自动通过remote_file存储到对称位置
|
144
|
-
|
145
|
-
todo 加进度条?方便超大文件下载查看进度。
|
146
|
-
|
147
|
-
使用示例:
|
148
|
-
sfc = SyncFileClient('http://yourserver.com', headers={"Authorization": "Bearer your_token"})
|
149
|
-
sfc.download_file('.vscode/launch.json', 'launch.json')
|
150
|
-
"""
|
151
|
-
if relpath:
|
152
|
-
local_file = self.get_abs_local_path(local_file, remote_file)
|
153
|
-
remote_file = self.get_abs_remote_path(remote_file, local_file)
|
154
|
-
|
155
|
-
data = {'file': remote_file.as_posix()}
|
156
|
-
# 使用 stream=True 开启流式处理
|
157
|
-
with requests.post(f'{self.host}/common/download_file',
|
158
|
-
headers=self.headers, json=data, stream=True) as resp:
|
159
|
-
if resp.status_code == 200:
|
160
|
-
# 以二进制方式写入文件
|
161
|
-
with open(local_file, 'wb') as f:
|
162
|
-
# 分块写入文件,每次 1MB (1024 * 1024 bytes)
|
163
|
-
for chunk in resp.iter_content(chunk_size=1024 * 1024):
|
164
|
-
if chunk: # 忽略keep-alive的空块
|
165
|
-
f.write(chunk)
|
166
|
-
return local_file
|
167
|
-
else:
|
168
|
-
# 处理其他状态码,抛出异常或进行错误处理
|
169
|
-
print(f"Error: Unable to download file, status code {resp.status_code}")
|
170
|
-
return None
|
171
|
-
|
172
|
-
def verify_etag(self, local_file=None, remote_file=None, relpath=True):
|
173
|
-
"""
|
174
|
-
:param relpath: 是否使用的是相对路径
|
175
|
-
当不使用relpath的时候,注意local_file、remote_file都是手动必填的XlPath类型字段
|
176
|
-
:return:
|
177
|
-
"""
|
178
|
-
if relpath:
|
179
|
-
local_file = self.get_abs_local_path(local_file, remote_file)
|
180
|
-
remote_file = self.get_abs_remote_path(remote_file, local_file)
|
181
|
-
if not local_file.is_file():
|
182
|
-
return {'match': False}
|
183
|
-
local_etag = GetEtag.from_file(local_file)
|
184
|
-
json_data = {'file': remote_file.as_posix(), 'etag': local_etag}
|
185
|
-
resp = requests.post(f'{self.host}/common/check_file_etag',
|
186
|
-
headers=self.headers,
|
187
|
-
json=json_data)
|
188
|
-
return resp.json()
|
189
|
-
|
190
|
-
def upload_dir(self,
|
191
|
-
local_dir=None,
|
192
|
-
remote_dir=None,
|
193
|
-
*,
|
194
|
-
delete_local_file=False,
|
195
|
-
verify_etag=False,
|
196
|
-
relpath=True):
|
197
|
-
"""
|
198
|
-
:param local_dir: 本地目录
|
199
|
-
:param remote_dir: 远程目录
|
200
|
-
:param delete_local_file: 每个文件上传成功后,是否删除本地文件
|
201
|
-
默认不删除,一般不删除是配合etag校验使用的
|
202
|
-
:param verify_etag: 上传前是否校验etag,如果服务器上对应位置已经有对应etag的文件,其实就不用上传了
|
203
|
-
|
204
|
-
该方法已修改以支持上传子目录和空目录
|
205
|
-
"""
|
206
|
-
if relpath:
|
207
|
-
# D:\home\chenkunze\data\temp2,举个具体的例子案例值,供参考理解
|
208
|
-
local_dir = self.get_abs_local_path(local_dir, remote_dir)
|
209
|
-
# /home/chenkunze/data/temp2
|
210
|
-
remote_dir = self.get_abs_remote_path(remote_dir, local_dir)
|
211
|
-
|
212
|
-
for root, dirs, files in os.walk(local_dir):
|
213
|
-
# root拿到的是这样的绝对路径:D:\home\chenkunze\data\temp2\数值问题
|
214
|
-
# '数值问题'
|
215
|
-
subdir = XlPath(root).relpath(local_dir)
|
216
|
-
if not files and not dirs:
|
217
|
-
# 如果该目录为空,则在远程服务器上创建空目录
|
218
|
-
self.create_remote_dir(remote_dir / subdir, relpath=False)
|
219
|
-
|
220
|
-
for file in tqdm(files, desc=f'上传目录:{root}'):
|
221
|
-
local_path = local_dir / subdir / file
|
222
|
-
remote_path = remote_dir / subdir / file
|
223
|
-
if verify_etag and self.verify_etag(local_path, remote_path, relpath=False):
|
224
|
-
pass # 如果etag校验通过,跳过上传
|
225
|
-
else:
|
226
|
-
self.upload_file(local_path, remote_path, relpath=False)
|
227
|
-
|
228
|
-
if delete_local_file:
|
229
|
-
os.remove(local_path)
|
230
|
-
|
231
|
-
def create_remote_dir(self, remote_dir, relpath=True):
|
232
|
-
"""
|
233
|
-
创建远程目录的辅助方法
|
234
|
-
"""
|
235
|
-
if relpath:
|
236
|
-
remote_dir = self.get_abs_remote_path(remote_dir)
|
237
|
-
json_data = {'dir': remote_dir.as_posix()}
|
238
|
-
resp = requests.post(f'{self.host}/common/create_dir',
|
239
|
-
headers=self.headers,
|
240
|
-
json=json_data)
|
241
|
-
return resp.json()
|
242
|
-
|
243
|
-
def download_dir(self, remote_dir=None, local_dir=None, *, verify_etag=False, relpath=True):
|
244
|
-
"""
|
245
|
-
下载远程目录到本地
|
246
|
-
:param remote_dir: 远程目录路径,相对路径或绝对路径都支持
|
247
|
-
:param local_dir: 本地保存目录,默认会自动映射到对应的远程位置
|
248
|
-
:param verify_etag: 是否进行etag校验,跳过已经在本地存在并且未修改的文件
|
249
|
-
:return:
|
250
|
-
|
251
|
-
示例:
|
252
|
-
sfc = SyncFileClient('codepc_mi15')
|
253
|
-
sfc.download_dir('data/temp2')
|
254
|
-
"""
|
255
|
-
# 1. 获取绝对路径
|
256
|
-
if relpath:
|
257
|
-
local_dir = self.get_abs_local_path(local_dir, remote_dir)
|
258
|
-
remote_dir = self.get_abs_remote_path(remote_dir, local_dir)
|
259
|
-
|
260
|
-
# 2. 从服务器获取远程目录的文件和子目录结构
|
261
|
-
json_data = {'dir': remote_dir.as_posix()}
|
262
|
-
resp = requests.post(f'{self.host}/common/list_dir', headers=self.headers, json=json_data)
|
263
|
-
if resp.status_code != 200:
|
264
|
-
raise Exception(f"Failed to list remote directory: {resp.text}")
|
265
|
-
|
266
|
-
dir_structure = resp.json() # 期望返回格式为 {'dirs': [...], 'files': [...]}
|
267
|
-
|
268
|
-
# 3. 创建本地目录
|
269
|
-
if not local_dir.exists():
|
270
|
-
local_dir.mkdir(parents=True)
|
271
|
-
|
272
|
-
# 4. 先处理子目录
|
273
|
-
for subdir in dir_structure.get('dirs', []):
|
274
|
-
local_subdir = local_dir / subdir
|
275
|
-
if not local_subdir.exists():
|
276
|
-
local_subdir.mkdir(parents=True)
|
277
|
-
|
278
|
-
# 递归下载子目录中的文件
|
279
|
-
for i in range(100): # 开发机senseserver3好像不稳定,加了重试机制
|
280
|
-
try:
|
281
|
-
self.download_dir(remote_dir / subdir, local_dir / subdir, verify_etag=verify_etag)
|
282
|
-
break
|
283
|
-
except requests.exceptions.RequestException as e:
|
284
|
-
logger.warning(format_exception(e, 3))
|
285
|
-
time.sleep(60)
|
286
|
-
continue
|
287
|
-
else:
|
288
|
-
raise requests.exceptions.RequestException
|
289
|
-
|
290
|
-
# 5. 下载文件
|
291
|
-
for file in tqdm(dir_structure.get('files', []), desc=f'下载目录:{remote_dir}'):
|
292
|
-
remote_file = remote_dir / file
|
293
|
-
local_file = local_dir / file
|
294
|
-
|
295
|
-
for i in range(100):
|
296
|
-
try:
|
297
|
-
# 校验etag,如果verify_etag为True并且校验成功,则跳过下载
|
298
|
-
if verify_etag and self.verify_etag(local_file, remote_file, relpath=False)['match']:
|
299
|
-
break
|
300
|
-
|
301
|
-
# 下载文件
|
302
|
-
logger.info(f'下载文件中:{local_file}')
|
303
|
-
self.download_file(remote_file, local_file, relpath=False)
|
304
|
-
break
|
305
|
-
except requests.exceptions.RequestException:
|
306
|
-
time.sleep(60)
|
307
|
-
continue
|
308
|
-
else:
|
309
|
-
raise requests.exceptions.RequestException
|
310
|
-
|
311
|
-
def download_path(self, remote_path=None, local_path=None, *, verify_etag=False, relpath=True):
|
312
|
-
"""
|
313
|
-
下载远程文件或目录到本地
|
314
|
-
:param remote_path: 远程路径,可以是文件或者目录
|
315
|
-
:param local_path: 本地保存路径,默认会自动映射到对应的远程位置
|
316
|
-
:param verify_etag: 是否进行etag校验,跳过已经在本地存在并且未修改的文件
|
317
|
-
:return:
|
318
|
-
"""
|
319
|
-
if relpath:
|
320
|
-
# 获取远程路径的绝对路径
|
321
|
-
remote_path = self.get_abs_remote_path(remote_path, local_path)
|
322
|
-
local_path = self.get_abs_local_path(local_path, remote_path)
|
323
|
-
|
324
|
-
# 判断是文件还是目录
|
325
|
-
json_data = {'path': remote_path.as_posix()}
|
326
|
-
resp = requests.post(f'{self.host}/common/check_path_type', headers=self.headers, json=json_data)
|
327
|
-
|
328
|
-
if resp.status_code != 200:
|
329
|
-
raise Exception(f"Failed to check remote path type: {resp.text}")
|
330
|
-
|
331
|
-
path_info = resp.json()
|
332
|
-
|
333
|
-
if path_info['type'] == 'file':
|
334
|
-
# 如果是文件,调用 download_file
|
335
|
-
self.download_file(remote_file=remote_path, local_file=local_path, relpath=False)
|
336
|
-
elif path_info['type'] == 'dir':
|
337
|
-
# 如果是目录,调用 download_dir
|
338
|
-
self.download_dir(remote_dir=remote_path, local_dir=local_path,
|
339
|
-
verify_etag=verify_etag, relpath=False)
|
340
|
-
else:
|
341
|
-
raise ValueError(f"Unknown path type: {path_info['type']}")
|
1
|
+
#!/usr/bin/env python3
|
2
|
+
# -*- coding: utf-8 -*-
|
3
|
+
# @Author : 陈坤泽
|
4
|
+
# @Email : 877362867@qq.com
|
5
|
+
# @Date : 2024/10/12
|
6
|
+
|
7
|
+
"""
|
8
|
+
我们自己的一套利用后端服务进行文件同步的工具
|
9
|
+
"""
|
10
|
+
|
11
|
+
import os
|
12
|
+
import time
|
13
|
+
|
14
|
+
from loguru import logger
|
15
|
+
from tqdm import tqdm
|
16
|
+
import requests
|
17
|
+
|
18
|
+
from pyxllib.file.specialist import XlPath, GetEtag
|
19
|
+
from pyxllib.prog.pupil import format_exception
|
20
|
+
|
21
|
+
|
22
|
+
class SyncFileClient:
|
23
|
+
"""
|
24
|
+
|
25
|
+
注意:该类填写的路径,一律是相对local_root、remote_root的路径,不是当前实际工作目录下的路径!
|
26
|
+
可以通过把local_host设为空的来实现一些特殊精细化的操作
|
27
|
+
但一般推荐还是都把local_root、remote_root设置上
|
28
|
+
"""
|
29
|
+
|
30
|
+
def __1_basic(self):
|
31
|
+
pass
|
32
|
+
|
33
|
+
def __init__(self, host, token=None):
|
34
|
+
"""
|
35
|
+
:param host: 可以只写主机名
|
36
|
+
"""
|
37
|
+
token = token or os.getenv("XL_COMMON_SERVER_TOKEN")
|
38
|
+
self.headers = {
|
39
|
+
'Authorization': f'Bearer {token}',
|
40
|
+
}
|
41
|
+
self.local_root = None
|
42
|
+
self.remote_root = None
|
43
|
+
self.host = self.link_host(host)
|
44
|
+
|
45
|
+
def link_host(self, hostname):
|
46
|
+
from requests.exceptions import Timeout
|
47
|
+
|
48
|
+
host = f'{os.getenv("MAIN_WEBSITE")}/{hostname}'
|
49
|
+
resp = requests.get(f'{host}/common/get_local_server', headers=self.headers)
|
50
|
+
ip = resp.json()['ip']
|
51
|
+
|
52
|
+
if ip:
|
53
|
+
try:
|
54
|
+
resp = requests.get(f'{ip}/healthz', timeout=5)
|
55
|
+
if resp.status_code == 200 and resp.json()['status'] == 'ok':
|
56
|
+
host = ip
|
57
|
+
except Timeout:
|
58
|
+
pass
|
59
|
+
except Exception as e:
|
60
|
+
raise e
|
61
|
+
|
62
|
+
remote_root = requests.get(f'{host}/common/get_wkdir', headers=self.headers).json()['wkdir']
|
63
|
+
self.remote_root = XlPath(remote_root)
|
64
|
+
|
65
|
+
return host
|
66
|
+
|
67
|
+
def get_abs_local_path(self, local_path, remote_path=None):
|
68
|
+
# 1 如果需要,从远程路径推测本地路径
|
69
|
+
if local_path is None and remote_path is not None:
|
70
|
+
local_path = remote_path
|
71
|
+
|
72
|
+
# 2 本地存储的文件路径
|
73
|
+
if self.local_root:
|
74
|
+
return self.local_root / local_path
|
75
|
+
else:
|
76
|
+
return XlPath(local_path)
|
77
|
+
|
78
|
+
def get_abs_remote_path(self, remote_path, local_path=None):
|
79
|
+
# 1 如果需要,从本地路径推测远程路径
|
80
|
+
if remote_path is None and local_path is not None:
|
81
|
+
remote_path = local_path
|
82
|
+
|
83
|
+
# 2 远程存储的文件路径
|
84
|
+
if self.remote_root:
|
85
|
+
return self.remote_root / remote_path
|
86
|
+
else:
|
87
|
+
return XlPath(remote_path)
|
88
|
+
|
89
|
+
def __2_sync(self):
|
90
|
+
pass
|
91
|
+
|
92
|
+
def upload_file(self, local_file=None, remote_file=None, relpath=True):
|
93
|
+
"""
|
94
|
+
:param str remote_file: 要存储的远程文件路径,默认可以不指定,自动通过local_file存储到对称位置
|
95
|
+
|
96
|
+
使用示例:
|
97
|
+
sfc = SyncFileClient('codepc_mi15')
|
98
|
+
sfc.upload_file('abc/README.md')
|
99
|
+
sfc.upload_file('abc/README.md', 'bcd/1.md')
|
100
|
+
"""
|
101
|
+
file = [local_file]
|
102
|
+
if remote_file:
|
103
|
+
file.append(remote_file)
|
104
|
+
return self.upload_files([file], relpath=relpath)
|
105
|
+
|
106
|
+
def upload_files(self, files, relpath=True):
|
107
|
+
"""
|
108
|
+
:param list[local_file<, remote_path>] files:
|
109
|
+
remote_path可选,未写的时候会自动填充
|
110
|
+
:return:
|
111
|
+
|
112
|
+
todo 加进度条?不过问了下gpt,这个实现好像有点复杂,先不折腾了。本来其实就推荐单个单个文件上传,在外部进行进度展示管控。
|
113
|
+
|
114
|
+
使用示例:
|
115
|
+
sfc = SyncFileClient('codepc_mi15')
|
116
|
+
print(sfc.upload_files(['data/d231120禅宗1期4阶.xlsx', 'data/d231120禅宗1期4阶2.xlsx']))
|
117
|
+
"""
|
118
|
+
# 1 映射出目标路径
|
119
|
+
files2 = []
|
120
|
+
for file in files:
|
121
|
+
if not isinstance(file, (list, tuple)):
|
122
|
+
local_file, remote_file = file, None
|
123
|
+
else:
|
124
|
+
local_file, remote_file = file
|
125
|
+
|
126
|
+
if relpath:
|
127
|
+
local_file = self.get_abs_local_path(local_file, remote_file)
|
128
|
+
remote_file = self.get_abs_remote_path(remote_file, local_file)
|
129
|
+
|
130
|
+
files2.append([local_file, remote_file.as_posix()])
|
131
|
+
|
132
|
+
files3 = [('files', (remote_file, open(local_file, 'rb'))) for local_file, remote_file in files2]
|
133
|
+
|
134
|
+
# 4 发送请求
|
135
|
+
resp = requests.post(f'{self.host}/common/upload_files',
|
136
|
+
headers=self.headers,
|
137
|
+
files=files3)
|
138
|
+
return resp.json()
|
139
|
+
|
140
|
+
def download_file(self, remote_file=None, local_file=None, relpath=True):
|
141
|
+
"""
|
142
|
+
:param str remote_file: 要下载的远程文件路径,相对get_wkdir下的路径
|
143
|
+
:param str local_file: 下载到本地的文件路径,默认可以不指定,自动通过remote_file存储到对称位置
|
144
|
+
|
145
|
+
todo 加进度条?方便超大文件下载查看进度。
|
146
|
+
|
147
|
+
使用示例:
|
148
|
+
sfc = SyncFileClient('http://yourserver.com', headers={"Authorization": "Bearer your_token"})
|
149
|
+
sfc.download_file('.vscode/launch.json', 'launch.json')
|
150
|
+
"""
|
151
|
+
if relpath:
|
152
|
+
local_file = self.get_abs_local_path(local_file, remote_file)
|
153
|
+
remote_file = self.get_abs_remote_path(remote_file, local_file)
|
154
|
+
|
155
|
+
data = {'file': remote_file.as_posix()}
|
156
|
+
# 使用 stream=True 开启流式处理
|
157
|
+
with requests.post(f'{self.host}/common/download_file',
|
158
|
+
headers=self.headers, json=data, stream=True) as resp:
|
159
|
+
if resp.status_code == 200:
|
160
|
+
# 以二进制方式写入文件
|
161
|
+
with open(local_file, 'wb') as f:
|
162
|
+
# 分块写入文件,每次 1MB (1024 * 1024 bytes)
|
163
|
+
for chunk in resp.iter_content(chunk_size=1024 * 1024):
|
164
|
+
if chunk: # 忽略keep-alive的空块
|
165
|
+
f.write(chunk)
|
166
|
+
return local_file
|
167
|
+
else:
|
168
|
+
# 处理其他状态码,抛出异常或进行错误处理
|
169
|
+
print(f"Error: Unable to download file, status code {resp.status_code}")
|
170
|
+
return None
|
171
|
+
|
172
|
+
def verify_etag(self, local_file=None, remote_file=None, relpath=True):
|
173
|
+
"""
|
174
|
+
:param relpath: 是否使用的是相对路径
|
175
|
+
当不使用relpath的时候,注意local_file、remote_file都是手动必填的XlPath类型字段
|
176
|
+
:return:
|
177
|
+
"""
|
178
|
+
if relpath:
|
179
|
+
local_file = self.get_abs_local_path(local_file, remote_file)
|
180
|
+
remote_file = self.get_abs_remote_path(remote_file, local_file)
|
181
|
+
if not local_file.is_file():
|
182
|
+
return {'match': False}
|
183
|
+
local_etag = GetEtag.from_file(local_file)
|
184
|
+
json_data = {'file': remote_file.as_posix(), 'etag': local_etag}
|
185
|
+
resp = requests.post(f'{self.host}/common/check_file_etag',
|
186
|
+
headers=self.headers,
|
187
|
+
json=json_data)
|
188
|
+
return resp.json()
|
189
|
+
|
190
|
+
def upload_dir(self,
|
191
|
+
local_dir=None,
|
192
|
+
remote_dir=None,
|
193
|
+
*,
|
194
|
+
delete_local_file=False,
|
195
|
+
verify_etag=False,
|
196
|
+
relpath=True):
|
197
|
+
"""
|
198
|
+
:param local_dir: 本地目录
|
199
|
+
:param remote_dir: 远程目录
|
200
|
+
:param delete_local_file: 每个文件上传成功后,是否删除本地文件
|
201
|
+
默认不删除,一般不删除是配合etag校验使用的
|
202
|
+
:param verify_etag: 上传前是否校验etag,如果服务器上对应位置已经有对应etag的文件,其实就不用上传了
|
203
|
+
|
204
|
+
该方法已修改以支持上传子目录和空目录
|
205
|
+
"""
|
206
|
+
if relpath:
|
207
|
+
# D:\home\chenkunze\data\temp2,举个具体的例子案例值,供参考理解
|
208
|
+
local_dir = self.get_abs_local_path(local_dir, remote_dir)
|
209
|
+
# /home/chenkunze/data/temp2
|
210
|
+
remote_dir = self.get_abs_remote_path(remote_dir, local_dir)
|
211
|
+
|
212
|
+
for root, dirs, files in os.walk(local_dir):
|
213
|
+
# root拿到的是这样的绝对路径:D:\home\chenkunze\data\temp2\数值问题
|
214
|
+
# '数值问题'
|
215
|
+
subdir = XlPath(root).relpath(local_dir)
|
216
|
+
if not files and not dirs:
|
217
|
+
# 如果该目录为空,则在远程服务器上创建空目录
|
218
|
+
self.create_remote_dir(remote_dir / subdir, relpath=False)
|
219
|
+
|
220
|
+
for file in tqdm(files, desc=f'上传目录:{root}'):
|
221
|
+
local_path = local_dir / subdir / file
|
222
|
+
remote_path = remote_dir / subdir / file
|
223
|
+
if verify_etag and self.verify_etag(local_path, remote_path, relpath=False):
|
224
|
+
pass # 如果etag校验通过,跳过上传
|
225
|
+
else:
|
226
|
+
self.upload_file(local_path, remote_path, relpath=False)
|
227
|
+
|
228
|
+
if delete_local_file:
|
229
|
+
os.remove(local_path)
|
230
|
+
|
231
|
+
def create_remote_dir(self, remote_dir, relpath=True):
|
232
|
+
"""
|
233
|
+
创建远程目录的辅助方法
|
234
|
+
"""
|
235
|
+
if relpath:
|
236
|
+
remote_dir = self.get_abs_remote_path(remote_dir)
|
237
|
+
json_data = {'dir': remote_dir.as_posix()}
|
238
|
+
resp = requests.post(f'{self.host}/common/create_dir',
|
239
|
+
headers=self.headers,
|
240
|
+
json=json_data)
|
241
|
+
return resp.json()
|
242
|
+
|
243
|
+
def download_dir(self, remote_dir=None, local_dir=None, *, verify_etag=False, relpath=True):
|
244
|
+
"""
|
245
|
+
下载远程目录到本地
|
246
|
+
:param remote_dir: 远程目录路径,相对路径或绝对路径都支持
|
247
|
+
:param local_dir: 本地保存目录,默认会自动映射到对应的远程位置
|
248
|
+
:param verify_etag: 是否进行etag校验,跳过已经在本地存在并且未修改的文件
|
249
|
+
:return:
|
250
|
+
|
251
|
+
示例:
|
252
|
+
sfc = SyncFileClient('codepc_mi15')
|
253
|
+
sfc.download_dir('data/temp2')
|
254
|
+
"""
|
255
|
+
# 1. 获取绝对路径
|
256
|
+
if relpath:
|
257
|
+
local_dir = self.get_abs_local_path(local_dir, remote_dir)
|
258
|
+
remote_dir = self.get_abs_remote_path(remote_dir, local_dir)
|
259
|
+
|
260
|
+
# 2. 从服务器获取远程目录的文件和子目录结构
|
261
|
+
json_data = {'dir': remote_dir.as_posix()}
|
262
|
+
resp = requests.post(f'{self.host}/common/list_dir', headers=self.headers, json=json_data)
|
263
|
+
if resp.status_code != 200:
|
264
|
+
raise Exception(f"Failed to list remote directory: {resp.text}")
|
265
|
+
|
266
|
+
dir_structure = resp.json() # 期望返回格式为 {'dirs': [...], 'files': [...]}
|
267
|
+
|
268
|
+
# 3. 创建本地目录
|
269
|
+
if not local_dir.exists():
|
270
|
+
local_dir.mkdir(parents=True)
|
271
|
+
|
272
|
+
# 4. 先处理子目录
|
273
|
+
for subdir in dir_structure.get('dirs', []):
|
274
|
+
local_subdir = local_dir / subdir
|
275
|
+
if not local_subdir.exists():
|
276
|
+
local_subdir.mkdir(parents=True)
|
277
|
+
|
278
|
+
# 递归下载子目录中的文件
|
279
|
+
for i in range(100): # 开发机senseserver3好像不稳定,加了重试机制
|
280
|
+
try:
|
281
|
+
self.download_dir(remote_dir / subdir, local_dir / subdir, verify_etag=verify_etag)
|
282
|
+
break
|
283
|
+
except requests.exceptions.RequestException as e:
|
284
|
+
logger.warning(format_exception(e, 3))
|
285
|
+
time.sleep(60)
|
286
|
+
continue
|
287
|
+
else:
|
288
|
+
raise requests.exceptions.RequestException
|
289
|
+
|
290
|
+
# 5. 下载文件
|
291
|
+
for file in tqdm(dir_structure.get('files', []), desc=f'下载目录:{remote_dir}'):
|
292
|
+
remote_file = remote_dir / file
|
293
|
+
local_file = local_dir / file
|
294
|
+
|
295
|
+
for i in range(100):
|
296
|
+
try:
|
297
|
+
# 校验etag,如果verify_etag为True并且校验成功,则跳过下载
|
298
|
+
if verify_etag and self.verify_etag(local_file, remote_file, relpath=False)['match']:
|
299
|
+
break
|
300
|
+
|
301
|
+
# 下载文件
|
302
|
+
logger.info(f'下载文件中:{local_file}')
|
303
|
+
self.download_file(remote_file, local_file, relpath=False)
|
304
|
+
break
|
305
|
+
except requests.exceptions.RequestException:
|
306
|
+
time.sleep(60)
|
307
|
+
continue
|
308
|
+
else:
|
309
|
+
raise requests.exceptions.RequestException
|
310
|
+
|
311
|
+
def download_path(self, remote_path=None, local_path=None, *, verify_etag=False, relpath=True):
|
312
|
+
"""
|
313
|
+
下载远程文件或目录到本地
|
314
|
+
:param remote_path: 远程路径,可以是文件或者目录
|
315
|
+
:param local_path: 本地保存路径,默认会自动映射到对应的远程位置
|
316
|
+
:param verify_etag: 是否进行etag校验,跳过已经在本地存在并且未修改的文件
|
317
|
+
:return:
|
318
|
+
"""
|
319
|
+
if relpath:
|
320
|
+
# 获取远程路径的绝对路径
|
321
|
+
remote_path = self.get_abs_remote_path(remote_path, local_path)
|
322
|
+
local_path = self.get_abs_local_path(local_path, remote_path)
|
323
|
+
|
324
|
+
# 判断是文件还是目录
|
325
|
+
json_data = {'path': remote_path.as_posix()}
|
326
|
+
resp = requests.post(f'{self.host}/common/check_path_type', headers=self.headers, json=json_data)
|
327
|
+
|
328
|
+
if resp.status_code != 200:
|
329
|
+
raise Exception(f"Failed to check remote path type: {resp.text}")
|
330
|
+
|
331
|
+
path_info = resp.json()
|
332
|
+
|
333
|
+
if path_info['type'] == 'file':
|
334
|
+
# 如果是文件,调用 download_file
|
335
|
+
self.download_file(remote_file=remote_path, local_file=local_path, relpath=False)
|
336
|
+
elif path_info['type'] == 'dir':
|
337
|
+
# 如果是目录,调用 download_dir
|
338
|
+
self.download_dir(remote_dir=remote_path, local_dir=local_path,
|
339
|
+
verify_etag=verify_etag, relpath=False)
|
340
|
+
else:
|
341
|
+
raise ValueError(f"Unknown path type: {path_info['type']}")
|