pyxllib 0.3.197__py3-none-any.whl → 3.201.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (127) hide show
  1. pyxllib/__init__.py +14 -21
  2. pyxllib/algo/__init__.py +8 -8
  3. pyxllib/algo/disjoint.py +54 -54
  4. pyxllib/algo/geo.py +537 -541
  5. pyxllib/algo/intervals.py +964 -964
  6. pyxllib/algo/matcher.py +389 -389
  7. pyxllib/algo/newbie.py +166 -166
  8. pyxllib/algo/pupil.py +629 -629
  9. pyxllib/algo/shapelylib.py +67 -67
  10. pyxllib/algo/specialist.py +241 -241
  11. pyxllib/algo/stat.py +494 -494
  12. pyxllib/algo/treelib.py +145 -149
  13. pyxllib/algo/unitlib.py +62 -66
  14. pyxllib/autogui/__init__.py +5 -5
  15. pyxllib/autogui/activewin.py +246 -246
  16. pyxllib/autogui/all.py +9 -9
  17. pyxllib/autogui/autogui.py +846 -852
  18. pyxllib/autogui/uiautolib.py +362 -362
  19. pyxllib/autogui/virtualkey.py +102 -102
  20. pyxllib/autogui/wechat.py +827 -827
  21. pyxllib/autogui/wechat_msg.py +421 -421
  22. pyxllib/autogui/wxautolib.py +84 -84
  23. pyxllib/cv/__init__.py +5 -5
  24. pyxllib/cv/expert.py +267 -267
  25. pyxllib/cv/imfile.py +159 -159
  26. pyxllib/cv/imhash.py +39 -39
  27. pyxllib/cv/pupil.py +9 -9
  28. pyxllib/cv/rgbfmt.py +1525 -1525
  29. pyxllib/cv/slidercaptcha.py +137 -137
  30. pyxllib/cv/trackbartools.py +251 -251
  31. pyxllib/cv/xlcvlib.py +1040 -1040
  32. pyxllib/cv/xlpillib.py +423 -423
  33. pyxllib/data/echarts.py +236 -240
  34. pyxllib/data/jsonlib.py +85 -89
  35. pyxllib/data/oss.py +72 -72
  36. pyxllib/data/pglib.py +1111 -1127
  37. pyxllib/data/sqlite.py +568 -568
  38. pyxllib/data/sqllib.py +297 -297
  39. pyxllib/ext/JLineViewer.py +505 -505
  40. pyxllib/ext/__init__.py +6 -6
  41. pyxllib/ext/demolib.py +251 -246
  42. pyxllib/ext/drissionlib.py +277 -277
  43. pyxllib/ext/kq5034lib.py +12 -12
  44. pyxllib/ext/qt.py +449 -449
  45. pyxllib/ext/robustprocfile.py +493 -497
  46. pyxllib/ext/seleniumlib.py +76 -76
  47. pyxllib/ext/tk.py +173 -173
  48. pyxllib/ext/unixlib.py +821 -827
  49. pyxllib/ext/utools.py +345 -351
  50. pyxllib/ext/webhook.py +124 -119
  51. pyxllib/ext/win32lib.py +40 -40
  52. pyxllib/ext/wjxlib.py +91 -88
  53. pyxllib/ext/wpsapi.py +124 -124
  54. pyxllib/ext/xlwork.py +9 -9
  55. pyxllib/ext/yuquelib.py +1110 -1105
  56. pyxllib/file/__init__.py +17 -17
  57. pyxllib/file/docxlib.py +757 -761
  58. pyxllib/file/gitlib.py +309 -309
  59. pyxllib/file/libreoffice.py +165 -165
  60. pyxllib/file/movielib.py +144 -148
  61. pyxllib/file/newbie.py +10 -10
  62. pyxllib/file/onenotelib.py +1469 -1469
  63. pyxllib/file/packlib/__init__.py +330 -330
  64. pyxllib/file/packlib/zipfile.py +2441 -2441
  65. pyxllib/file/pdflib.py +422 -426
  66. pyxllib/file/pupil.py +185 -185
  67. pyxllib/file/specialist/__init__.py +681 -685
  68. pyxllib/file/specialist/dirlib.py +799 -799
  69. pyxllib/file/specialist/download.py +193 -193
  70. pyxllib/file/specialist/filelib.py +2825 -2829
  71. pyxllib/file/xlsxlib.py +3122 -3131
  72. pyxllib/file/xlsyncfile.py +341 -341
  73. pyxllib/prog/__init__.py +5 -5
  74. pyxllib/prog/cachetools.py +58 -64
  75. pyxllib/prog/deprecatedlib.py +233 -233
  76. pyxllib/prog/filelock.py +42 -42
  77. pyxllib/prog/ipyexec.py +253 -253
  78. pyxllib/prog/multiprogs.py +940 -940
  79. pyxllib/prog/newbie.py +451 -451
  80. pyxllib/prog/pupil.py +1208 -1197
  81. pyxllib/prog/sitepackages.py +33 -33
  82. pyxllib/prog/specialist/__init__.py +348 -391
  83. pyxllib/prog/specialist/bc.py +203 -203
  84. pyxllib/prog/specialist/browser.py +497 -497
  85. pyxllib/prog/specialist/common.py +347 -347
  86. pyxllib/prog/specialist/datetime.py +198 -198
  87. pyxllib/prog/specialist/tictoc.py +240 -240
  88. pyxllib/prog/specialist/xllog.py +180 -180
  89. pyxllib/prog/xlosenv.py +110 -108
  90. pyxllib/stdlib/__init__.py +17 -17
  91. pyxllib/stdlib/tablepyxl/__init__.py +10 -10
  92. pyxllib/stdlib/tablepyxl/style.py +303 -303
  93. pyxllib/stdlib/tablepyxl/tablepyxl.py +130 -130
  94. pyxllib/text/__init__.py +8 -8
  95. pyxllib/text/ahocorasick.py +36 -39
  96. pyxllib/text/airscript.js +754 -744
  97. pyxllib/text/charclasslib.py +121 -121
  98. pyxllib/text/jiebalib.py +267 -267
  99. pyxllib/text/jinjalib.py +27 -32
  100. pyxllib/text/jsa_ai_prompt.md +271 -271
  101. pyxllib/text/jscode.py +922 -922
  102. pyxllib/text/latex/__init__.py +158 -158
  103. pyxllib/text/levenshtein.py +303 -303
  104. pyxllib/text/nestenv.py +1215 -1215
  105. pyxllib/text/newbie.py +300 -300
  106. pyxllib/text/pupil/__init__.py +8 -8
  107. pyxllib/text/pupil/common.py +1121 -1121
  108. pyxllib/text/pupil/xlalign.py +326 -326
  109. pyxllib/text/pycode.py +47 -47
  110. pyxllib/text/specialist/__init__.py +8 -8
  111. pyxllib/text/specialist/common.py +112 -112
  112. pyxllib/text/specialist/ptag.py +186 -186
  113. pyxllib/text/spellchecker.py +172 -172
  114. pyxllib/text/templates/echart_base.html +10 -10
  115. pyxllib/text/templates/highlight_code.html +16 -16
  116. pyxllib/text/templates/latex_editor.html +102 -102
  117. pyxllib/text/vbacode.py +17 -17
  118. pyxllib/text/xmllib.py +741 -747
  119. pyxllib/xl.py +42 -39
  120. pyxllib/xlcv.py +17 -17
  121. pyxllib-3.201.1.dist-info/METADATA +296 -0
  122. pyxllib-3.201.1.dist-info/RECORD +125 -0
  123. {pyxllib-0.3.197.dist-info → pyxllib-3.201.1.dist-info}/licenses/LICENSE +190 -190
  124. pyxllib/ext/old.py +0 -663
  125. pyxllib-0.3.197.dist-info/METADATA +0 -48
  126. pyxllib-0.3.197.dist-info/RECORD +0 -126
  127. {pyxllib-0.3.197.dist-info → pyxllib-3.201.1.dist-info}/WHEEL +0 -0
@@ -1,341 +1,341 @@
1
- #!/usr/bin/env python3
2
- # -*- coding: utf-8 -*-
3
- # @Author : 陈坤泽
4
- # @Email : 877362867@qq.com
5
- # @Date : 2024/10/12
6
-
7
- """
8
- 我们自己的一套利用后端服务进行文件同步的工具
9
- """
10
-
11
- import os
12
- import time
13
-
14
- from loguru import logger
15
- from tqdm import tqdm
16
- import requests
17
-
18
- from pyxllib.file.specialist import XlPath, GetEtag
19
- from pyxllib.prog.pupil import format_exception
20
-
21
-
22
- class SyncFileClient:
23
- """
24
-
25
- 注意:该类填写的路径,一律是相对local_root、remote_root的路径,不是当前实际工作目录下的路径!
26
- 可以通过把local_host设为空的来实现一些特殊精细化的操作
27
- 但一般推荐还是都把local_root、remote_root设置上
28
- """
29
-
30
- def __1_basic(self):
31
- pass
32
-
33
- def __init__(self, host, token=None):
34
- """
35
- :param host: 可以只写主机名
36
- """
37
- token = token or os.getenv("XL_COMMON_SERVER_TOKEN")
38
- self.headers = {
39
- 'Authorization': f'Bearer {token}',
40
- }
41
- self.local_root = None
42
- self.remote_root = None
43
- self.host = self.link_host(host)
44
-
45
- def link_host(self, hostname):
46
- from requests.exceptions import Timeout
47
-
48
- host = f'{os.getenv("MAIN_WEBSITE")}/{hostname}'
49
- resp = requests.get(f'{host}/common/get_local_server', headers=self.headers)
50
- ip = resp.json()['ip']
51
-
52
- if ip:
53
- try:
54
- resp = requests.get(f'{ip}/healthz', timeout=5)
55
- if resp.status_code == 200 and resp.json()['status'] == 'ok':
56
- host = ip
57
- except Timeout:
58
- pass
59
- except Exception as e:
60
- raise e
61
-
62
- remote_root = requests.get(f'{host}/common/get_wkdir', headers=self.headers).json()['wkdir']
63
- self.remote_root = XlPath(remote_root)
64
-
65
- return host
66
-
67
- def get_abs_local_path(self, local_path, remote_path=None):
68
- # 1 如果需要,从远程路径推测本地路径
69
- if local_path is None and remote_path is not None:
70
- local_path = remote_path
71
-
72
- # 2 本地存储的文件路径
73
- if self.local_root:
74
- return self.local_root / local_path
75
- else:
76
- return XlPath(local_path)
77
-
78
- def get_abs_remote_path(self, remote_path, local_path=None):
79
- # 1 如果需要,从本地路径推测远程路径
80
- if remote_path is None and local_path is not None:
81
- remote_path = local_path
82
-
83
- # 2 远程存储的文件路径
84
- if self.remote_root:
85
- return self.remote_root / remote_path
86
- else:
87
- return XlPath(remote_path)
88
-
89
- def __2_sync(self):
90
- pass
91
-
92
- def upload_file(self, local_file=None, remote_file=None, relpath=True):
93
- """
94
- :param str remote_file: 要存储的远程文件路径,默认可以不指定,自动通过local_file存储到对称位置
95
-
96
- 使用示例:
97
- sfc = SyncFileClient('codepc_mi15')
98
- sfc.upload_file('abc/README.md')
99
- sfc.upload_file('abc/README.md', 'bcd/1.md')
100
- """
101
- file = [local_file]
102
- if remote_file:
103
- file.append(remote_file)
104
- return self.upload_files([file], relpath=relpath)
105
-
106
- def upload_files(self, files, relpath=True):
107
- """
108
- :param list[local_file<, remote_path>] files:
109
- remote_path可选,未写的时候会自动填充
110
- :return:
111
-
112
- todo 加进度条?不过问了下gpt,这个实现好像有点复杂,先不折腾了。本来其实就推荐单个单个文件上传,在外部进行进度展示管控。
113
-
114
- 使用示例:
115
- sfc = SyncFileClient('codepc_mi15')
116
- print(sfc.upload_files(['data/d231120禅宗1期4阶.xlsx', 'data/d231120禅宗1期4阶2.xlsx']))
117
- """
118
- # 1 映射出目标路径
119
- files2 = []
120
- for file in files:
121
- if not isinstance(file, (list, tuple)):
122
- local_file, remote_file = file, None
123
- else:
124
- local_file, remote_file = file
125
-
126
- if relpath:
127
- local_file = self.get_abs_local_path(local_file, remote_file)
128
- remote_file = self.get_abs_remote_path(remote_file, local_file)
129
-
130
- files2.append([local_file, remote_file.as_posix()])
131
-
132
- files3 = [('files', (remote_file, open(local_file, 'rb'))) for local_file, remote_file in files2]
133
-
134
- # 4 发送请求
135
- resp = requests.post(f'{self.host}/common/upload_files',
136
- headers=self.headers,
137
- files=files3)
138
- return resp.json()
139
-
140
- def download_file(self, remote_file=None, local_file=None, relpath=True):
141
- """
142
- :param str remote_file: 要下载的远程文件路径,相对get_wkdir下的路径
143
- :param str local_file: 下载到本地的文件路径,默认可以不指定,自动通过remote_file存储到对称位置
144
-
145
- todo 加进度条?方便超大文件下载查看进度。
146
-
147
- 使用示例:
148
- sfc = SyncFileClient('http://yourserver.com', headers={"Authorization": "Bearer your_token"})
149
- sfc.download_file('.vscode/launch.json', 'launch.json')
150
- """
151
- if relpath:
152
- local_file = self.get_abs_local_path(local_file, remote_file)
153
- remote_file = self.get_abs_remote_path(remote_file, local_file)
154
-
155
- data = {'file': remote_file.as_posix()}
156
- # 使用 stream=True 开启流式处理
157
- with requests.post(f'{self.host}/common/download_file',
158
- headers=self.headers, json=data, stream=True) as resp:
159
- if resp.status_code == 200:
160
- # 以二进制方式写入文件
161
- with open(local_file, 'wb') as f:
162
- # 分块写入文件,每次 1MB (1024 * 1024 bytes)
163
- for chunk in resp.iter_content(chunk_size=1024 * 1024):
164
- if chunk: # 忽略keep-alive的空块
165
- f.write(chunk)
166
- return local_file
167
- else:
168
- # 处理其他状态码,抛出异常或进行错误处理
169
- print(f"Error: Unable to download file, status code {resp.status_code}")
170
- return None
171
-
172
- def verify_etag(self, local_file=None, remote_file=None, relpath=True):
173
- """
174
- :param relpath: 是否使用的是相对路径
175
- 当不使用relpath的时候,注意local_file、remote_file都是手动必填的XlPath类型字段
176
- :return:
177
- """
178
- if relpath:
179
- local_file = self.get_abs_local_path(local_file, remote_file)
180
- remote_file = self.get_abs_remote_path(remote_file, local_file)
181
- if not local_file.is_file():
182
- return {'match': False}
183
- local_etag = GetEtag.from_file(local_file)
184
- json_data = {'file': remote_file.as_posix(), 'etag': local_etag}
185
- resp = requests.post(f'{self.host}/common/check_file_etag',
186
- headers=self.headers,
187
- json=json_data)
188
- return resp.json()
189
-
190
- def upload_dir(self,
191
- local_dir=None,
192
- remote_dir=None,
193
- *,
194
- delete_local_file=False,
195
- verify_etag=False,
196
- relpath=True):
197
- """
198
- :param local_dir: 本地目录
199
- :param remote_dir: 远程目录
200
- :param delete_local_file: 每个文件上传成功后,是否删除本地文件
201
- 默认不删除,一般不删除是配合etag校验使用的
202
- :param verify_etag: 上传前是否校验etag,如果服务器上对应位置已经有对应etag的文件,其实就不用上传了
203
-
204
- 该方法已修改以支持上传子目录和空目录
205
- """
206
- if relpath:
207
- # D:\home\chenkunze\data\temp2,举个具体的例子案例值,供参考理解
208
- local_dir = self.get_abs_local_path(local_dir, remote_dir)
209
- # /home/chenkunze/data/temp2
210
- remote_dir = self.get_abs_remote_path(remote_dir, local_dir)
211
-
212
- for root, dirs, files in os.walk(local_dir):
213
- # root拿到的是这样的绝对路径:D:\home\chenkunze\data\temp2\数值问题
214
- # '数值问题'
215
- subdir = XlPath(root).relpath(local_dir)
216
- if not files and not dirs:
217
- # 如果该目录为空,则在远程服务器上创建空目录
218
- self.create_remote_dir(remote_dir / subdir, relpath=False)
219
-
220
- for file in tqdm(files, desc=f'上传目录:{root}'):
221
- local_path = local_dir / subdir / file
222
- remote_path = remote_dir / subdir / file
223
- if verify_etag and self.verify_etag(local_path, remote_path, relpath=False):
224
- pass # 如果etag校验通过,跳过上传
225
- else:
226
- self.upload_file(local_path, remote_path, relpath=False)
227
-
228
- if delete_local_file:
229
- os.remove(local_path)
230
-
231
- def create_remote_dir(self, remote_dir, relpath=True):
232
- """
233
- 创建远程目录的辅助方法
234
- """
235
- if relpath:
236
- remote_dir = self.get_abs_remote_path(remote_dir)
237
- json_data = {'dir': remote_dir.as_posix()}
238
- resp = requests.post(f'{self.host}/common/create_dir',
239
- headers=self.headers,
240
- json=json_data)
241
- return resp.json()
242
-
243
- def download_dir(self, remote_dir=None, local_dir=None, *, verify_etag=False, relpath=True):
244
- """
245
- 下载远程目录到本地
246
- :param remote_dir: 远程目录路径,相对路径或绝对路径都支持
247
- :param local_dir: 本地保存目录,默认会自动映射到对应的远程位置
248
- :param verify_etag: 是否进行etag校验,跳过已经在本地存在并且未修改的文件
249
- :return:
250
-
251
- 示例:
252
- sfc = SyncFileClient('codepc_mi15')
253
- sfc.download_dir('data/temp2')
254
- """
255
- # 1. 获取绝对路径
256
- if relpath:
257
- local_dir = self.get_abs_local_path(local_dir, remote_dir)
258
- remote_dir = self.get_abs_remote_path(remote_dir, local_dir)
259
-
260
- # 2. 从服务器获取远程目录的文件和子目录结构
261
- json_data = {'dir': remote_dir.as_posix()}
262
- resp = requests.post(f'{self.host}/common/list_dir', headers=self.headers, json=json_data)
263
- if resp.status_code != 200:
264
- raise Exception(f"Failed to list remote directory: {resp.text}")
265
-
266
- dir_structure = resp.json() # 期望返回格式为 {'dirs': [...], 'files': [...]}
267
-
268
- # 3. 创建本地目录
269
- if not local_dir.exists():
270
- local_dir.mkdir(parents=True)
271
-
272
- # 4. 先处理子目录
273
- for subdir in dir_structure.get('dirs', []):
274
- local_subdir = local_dir / subdir
275
- if not local_subdir.exists():
276
- local_subdir.mkdir(parents=True)
277
-
278
- # 递归下载子目录中的文件
279
- for i in range(100): # 开发机senseserver3好像不稳定,加了重试机制
280
- try:
281
- self.download_dir(remote_dir / subdir, local_dir / subdir, verify_etag=verify_etag)
282
- break
283
- except requests.exceptions.RequestException as e:
284
- logger.warning(format_exception(e, 3))
285
- time.sleep(60)
286
- continue
287
- else:
288
- raise requests.exceptions.RequestException
289
-
290
- # 5. 下载文件
291
- for file in tqdm(dir_structure.get('files', []), desc=f'下载目录:{remote_dir}'):
292
- remote_file = remote_dir / file
293
- local_file = local_dir / file
294
-
295
- for i in range(100):
296
- try:
297
- # 校验etag,如果verify_etag为True并且校验成功,则跳过下载
298
- if verify_etag and self.verify_etag(local_file, remote_file, relpath=False)['match']:
299
- break
300
-
301
- # 下载文件
302
- logger.info(f'下载文件中:{local_file}')
303
- self.download_file(remote_file, local_file, relpath=False)
304
- break
305
- except requests.exceptions.RequestException:
306
- time.sleep(60)
307
- continue
308
- else:
309
- raise requests.exceptions.RequestException
310
-
311
- def download_path(self, remote_path=None, local_path=None, *, verify_etag=False, relpath=True):
312
- """
313
- 下载远程文件或目录到本地
314
- :param remote_path: 远程路径,可以是文件或者目录
315
- :param local_path: 本地保存路径,默认会自动映射到对应的远程位置
316
- :param verify_etag: 是否进行etag校验,跳过已经在本地存在并且未修改的文件
317
- :return:
318
- """
319
- if relpath:
320
- # 获取远程路径的绝对路径
321
- remote_path = self.get_abs_remote_path(remote_path, local_path)
322
- local_path = self.get_abs_local_path(local_path, remote_path)
323
-
324
- # 判断是文件还是目录
325
- json_data = {'path': remote_path.as_posix()}
326
- resp = requests.post(f'{self.host}/common/check_path_type', headers=self.headers, json=json_data)
327
-
328
- if resp.status_code != 200:
329
- raise Exception(f"Failed to check remote path type: {resp.text}")
330
-
331
- path_info = resp.json()
332
-
333
- if path_info['type'] == 'file':
334
- # 如果是文件,调用 download_file
335
- self.download_file(remote_file=remote_path, local_file=local_path, relpath=False)
336
- elif path_info['type'] == 'dir':
337
- # 如果是目录,调用 download_dir
338
- self.download_dir(remote_dir=remote_path, local_dir=local_path,
339
- verify_etag=verify_etag, relpath=False)
340
- else:
341
- raise ValueError(f"Unknown path type: {path_info['type']}")
1
+ #!/usr/bin/env python3
2
+ # -*- coding: utf-8 -*-
3
+ # @Author : 陈坤泽
4
+ # @Email : 877362867@qq.com
5
+ # @Date : 2024/10/12
6
+
7
+ """
8
+ 我们自己的一套利用后端服务进行文件同步的工具
9
+ """
10
+
11
+ import os
12
+ import time
13
+
14
+ from loguru import logger
15
+ from tqdm import tqdm
16
+ import requests
17
+
18
+ from pyxllib.file.specialist import XlPath, GetEtag
19
+ from pyxllib.prog.pupil import format_exception
20
+
21
+
22
+ class SyncFileClient:
23
+ """
24
+
25
+ 注意:该类填写的路径,一律是相对local_root、remote_root的路径,不是当前实际工作目录下的路径!
26
+ 可以通过把local_host设为空的来实现一些特殊精细化的操作
27
+ 但一般推荐还是都把local_root、remote_root设置上
28
+ """
29
+
30
+ def __1_basic(self):
31
+ pass
32
+
33
+ def __init__(self, host, token=None):
34
+ """
35
+ :param host: 可以只写主机名
36
+ """
37
+ token = token or os.getenv("XL_COMMON_SERVER_TOKEN")
38
+ self.headers = {
39
+ 'Authorization': f'Bearer {token}',
40
+ }
41
+ self.local_root = None
42
+ self.remote_root = None
43
+ self.host = self.link_host(host)
44
+
45
+ def link_host(self, hostname):
46
+ from requests.exceptions import Timeout
47
+
48
+ host = f'{os.getenv("MAIN_WEBSITE")}/{hostname}'
49
+ resp = requests.get(f'{host}/common/get_local_server', headers=self.headers)
50
+ ip = resp.json()['ip']
51
+
52
+ if ip:
53
+ try:
54
+ resp = requests.get(f'{ip}/healthz', timeout=5)
55
+ if resp.status_code == 200 and resp.json()['status'] == 'ok':
56
+ host = ip
57
+ except Timeout:
58
+ pass
59
+ except Exception as e:
60
+ raise e
61
+
62
+ remote_root = requests.get(f'{host}/common/get_wkdir', headers=self.headers).json()['wkdir']
63
+ self.remote_root = XlPath(remote_root)
64
+
65
+ return host
66
+
67
+ def get_abs_local_path(self, local_path, remote_path=None):
68
+ # 1 如果需要,从远程路径推测本地路径
69
+ if local_path is None and remote_path is not None:
70
+ local_path = remote_path
71
+
72
+ # 2 本地存储的文件路径
73
+ if self.local_root:
74
+ return self.local_root / local_path
75
+ else:
76
+ return XlPath(local_path)
77
+
78
+ def get_abs_remote_path(self, remote_path, local_path=None):
79
+ # 1 如果需要,从本地路径推测远程路径
80
+ if remote_path is None and local_path is not None:
81
+ remote_path = local_path
82
+
83
+ # 2 远程存储的文件路径
84
+ if self.remote_root:
85
+ return self.remote_root / remote_path
86
+ else:
87
+ return XlPath(remote_path)
88
+
89
+ def __2_sync(self):
90
+ pass
91
+
92
+ def upload_file(self, local_file=None, remote_file=None, relpath=True):
93
+ """
94
+ :param str remote_file: 要存储的远程文件路径,默认可以不指定,自动通过local_file存储到对称位置
95
+
96
+ 使用示例:
97
+ sfc = SyncFileClient('codepc_mi15')
98
+ sfc.upload_file('abc/README.md')
99
+ sfc.upload_file('abc/README.md', 'bcd/1.md')
100
+ """
101
+ file = [local_file]
102
+ if remote_file:
103
+ file.append(remote_file)
104
+ return self.upload_files([file], relpath=relpath)
105
+
106
+ def upload_files(self, files, relpath=True):
107
+ """
108
+ :param list[local_file<, remote_path>] files:
109
+ remote_path可选,未写的时候会自动填充
110
+ :return:
111
+
112
+ todo 加进度条?不过问了下gpt,这个实现好像有点复杂,先不折腾了。本来其实就推荐单个单个文件上传,在外部进行进度展示管控。
113
+
114
+ 使用示例:
115
+ sfc = SyncFileClient('codepc_mi15')
116
+ print(sfc.upload_files(['data/d231120禅宗1期4阶.xlsx', 'data/d231120禅宗1期4阶2.xlsx']))
117
+ """
118
+ # 1 映射出目标路径
119
+ files2 = []
120
+ for file in files:
121
+ if not isinstance(file, (list, tuple)):
122
+ local_file, remote_file = file, None
123
+ else:
124
+ local_file, remote_file = file
125
+
126
+ if relpath:
127
+ local_file = self.get_abs_local_path(local_file, remote_file)
128
+ remote_file = self.get_abs_remote_path(remote_file, local_file)
129
+
130
+ files2.append([local_file, remote_file.as_posix()])
131
+
132
+ files3 = [('files', (remote_file, open(local_file, 'rb'))) for local_file, remote_file in files2]
133
+
134
+ # 4 发送请求
135
+ resp = requests.post(f'{self.host}/common/upload_files',
136
+ headers=self.headers,
137
+ files=files3)
138
+ return resp.json()
139
+
140
+ def download_file(self, remote_file=None, local_file=None, relpath=True):
141
+ """
142
+ :param str remote_file: 要下载的远程文件路径,相对get_wkdir下的路径
143
+ :param str local_file: 下载到本地的文件路径,默认可以不指定,自动通过remote_file存储到对称位置
144
+
145
+ todo 加进度条?方便超大文件下载查看进度。
146
+
147
+ 使用示例:
148
+ sfc = SyncFileClient('http://yourserver.com', headers={"Authorization": "Bearer your_token"})
149
+ sfc.download_file('.vscode/launch.json', 'launch.json')
150
+ """
151
+ if relpath:
152
+ local_file = self.get_abs_local_path(local_file, remote_file)
153
+ remote_file = self.get_abs_remote_path(remote_file, local_file)
154
+
155
+ data = {'file': remote_file.as_posix()}
156
+ # 使用 stream=True 开启流式处理
157
+ with requests.post(f'{self.host}/common/download_file',
158
+ headers=self.headers, json=data, stream=True) as resp:
159
+ if resp.status_code == 200:
160
+ # 以二进制方式写入文件
161
+ with open(local_file, 'wb') as f:
162
+ # 分块写入文件,每次 1MB (1024 * 1024 bytes)
163
+ for chunk in resp.iter_content(chunk_size=1024 * 1024):
164
+ if chunk: # 忽略keep-alive的空块
165
+ f.write(chunk)
166
+ return local_file
167
+ else:
168
+ # 处理其他状态码,抛出异常或进行错误处理
169
+ print(f"Error: Unable to download file, status code {resp.status_code}")
170
+ return None
171
+
172
+ def verify_etag(self, local_file=None, remote_file=None, relpath=True):
173
+ """
174
+ :param relpath: 是否使用的是相对路径
175
+ 当不使用relpath的时候,注意local_file、remote_file都是手动必填的XlPath类型字段
176
+ :return:
177
+ """
178
+ if relpath:
179
+ local_file = self.get_abs_local_path(local_file, remote_file)
180
+ remote_file = self.get_abs_remote_path(remote_file, local_file)
181
+ if not local_file.is_file():
182
+ return {'match': False}
183
+ local_etag = GetEtag.from_file(local_file)
184
+ json_data = {'file': remote_file.as_posix(), 'etag': local_etag}
185
+ resp = requests.post(f'{self.host}/common/check_file_etag',
186
+ headers=self.headers,
187
+ json=json_data)
188
+ return resp.json()
189
+
190
+ def upload_dir(self,
191
+ local_dir=None,
192
+ remote_dir=None,
193
+ *,
194
+ delete_local_file=False,
195
+ verify_etag=False,
196
+ relpath=True):
197
+ """
198
+ :param local_dir: 本地目录
199
+ :param remote_dir: 远程目录
200
+ :param delete_local_file: 每个文件上传成功后,是否删除本地文件
201
+ 默认不删除,一般不删除是配合etag校验使用的
202
+ :param verify_etag: 上传前是否校验etag,如果服务器上对应位置已经有对应etag的文件,其实就不用上传了
203
+
204
+ 该方法已修改以支持上传子目录和空目录
205
+ """
206
+ if relpath:
207
+ # D:\home\chenkunze\data\temp2,举个具体的例子案例值,供参考理解
208
+ local_dir = self.get_abs_local_path(local_dir, remote_dir)
209
+ # /home/chenkunze/data/temp2
210
+ remote_dir = self.get_abs_remote_path(remote_dir, local_dir)
211
+
212
+ for root, dirs, files in os.walk(local_dir):
213
+ # root拿到的是这样的绝对路径:D:\home\chenkunze\data\temp2\数值问题
214
+ # '数值问题'
215
+ subdir = XlPath(root).relpath(local_dir)
216
+ if not files and not dirs:
217
+ # 如果该目录为空,则在远程服务器上创建空目录
218
+ self.create_remote_dir(remote_dir / subdir, relpath=False)
219
+
220
+ for file in tqdm(files, desc=f'上传目录:{root}'):
221
+ local_path = local_dir / subdir / file
222
+ remote_path = remote_dir / subdir / file
223
+ if verify_etag and self.verify_etag(local_path, remote_path, relpath=False):
224
+ pass # 如果etag校验通过,跳过上传
225
+ else:
226
+ self.upload_file(local_path, remote_path, relpath=False)
227
+
228
+ if delete_local_file:
229
+ os.remove(local_path)
230
+
231
+ def create_remote_dir(self, remote_dir, relpath=True):
232
+ """
233
+ 创建远程目录的辅助方法
234
+ """
235
+ if relpath:
236
+ remote_dir = self.get_abs_remote_path(remote_dir)
237
+ json_data = {'dir': remote_dir.as_posix()}
238
+ resp = requests.post(f'{self.host}/common/create_dir',
239
+ headers=self.headers,
240
+ json=json_data)
241
+ return resp.json()
242
+
243
+ def download_dir(self, remote_dir=None, local_dir=None, *, verify_etag=False, relpath=True):
244
+ """
245
+ 下载远程目录到本地
246
+ :param remote_dir: 远程目录路径,相对路径或绝对路径都支持
247
+ :param local_dir: 本地保存目录,默认会自动映射到对应的远程位置
248
+ :param verify_etag: 是否进行etag校验,跳过已经在本地存在并且未修改的文件
249
+ :return:
250
+
251
+ 示例:
252
+ sfc = SyncFileClient('codepc_mi15')
253
+ sfc.download_dir('data/temp2')
254
+ """
255
+ # 1. 获取绝对路径
256
+ if relpath:
257
+ local_dir = self.get_abs_local_path(local_dir, remote_dir)
258
+ remote_dir = self.get_abs_remote_path(remote_dir, local_dir)
259
+
260
+ # 2. 从服务器获取远程目录的文件和子目录结构
261
+ json_data = {'dir': remote_dir.as_posix()}
262
+ resp = requests.post(f'{self.host}/common/list_dir', headers=self.headers, json=json_data)
263
+ if resp.status_code != 200:
264
+ raise Exception(f"Failed to list remote directory: {resp.text}")
265
+
266
+ dir_structure = resp.json() # 期望返回格式为 {'dirs': [...], 'files': [...]}
267
+
268
+ # 3. 创建本地目录
269
+ if not local_dir.exists():
270
+ local_dir.mkdir(parents=True)
271
+
272
+ # 4. 先处理子目录
273
+ for subdir in dir_structure.get('dirs', []):
274
+ local_subdir = local_dir / subdir
275
+ if not local_subdir.exists():
276
+ local_subdir.mkdir(parents=True)
277
+
278
+ # 递归下载子目录中的文件
279
+ for i in range(100): # 开发机senseserver3好像不稳定,加了重试机制
280
+ try:
281
+ self.download_dir(remote_dir / subdir, local_dir / subdir, verify_etag=verify_etag)
282
+ break
283
+ except requests.exceptions.RequestException as e:
284
+ logger.warning(format_exception(e, 3))
285
+ time.sleep(60)
286
+ continue
287
+ else:
288
+ raise requests.exceptions.RequestException
289
+
290
+ # 5. 下载文件
291
+ for file in tqdm(dir_structure.get('files', []), desc=f'下载目录:{remote_dir}'):
292
+ remote_file = remote_dir / file
293
+ local_file = local_dir / file
294
+
295
+ for i in range(100):
296
+ try:
297
+ # 校验etag,如果verify_etag为True并且校验成功,则跳过下载
298
+ if verify_etag and self.verify_etag(local_file, remote_file, relpath=False)['match']:
299
+ break
300
+
301
+ # 下载文件
302
+ logger.info(f'下载文件中:{local_file}')
303
+ self.download_file(remote_file, local_file, relpath=False)
304
+ break
305
+ except requests.exceptions.RequestException:
306
+ time.sleep(60)
307
+ continue
308
+ else:
309
+ raise requests.exceptions.RequestException
310
+
311
+ def download_path(self, remote_path=None, local_path=None, *, verify_etag=False, relpath=True):
312
+ """
313
+ 下载远程文件或目录到本地
314
+ :param remote_path: 远程路径,可以是文件或者目录
315
+ :param local_path: 本地保存路径,默认会自动映射到对应的远程位置
316
+ :param verify_etag: 是否进行etag校验,跳过已经在本地存在并且未修改的文件
317
+ :return:
318
+ """
319
+ if relpath:
320
+ # 获取远程路径的绝对路径
321
+ remote_path = self.get_abs_remote_path(remote_path, local_path)
322
+ local_path = self.get_abs_local_path(local_path, remote_path)
323
+
324
+ # 判断是文件还是目录
325
+ json_data = {'path': remote_path.as_posix()}
326
+ resp = requests.post(f'{self.host}/common/check_path_type', headers=self.headers, json=json_data)
327
+
328
+ if resp.status_code != 200:
329
+ raise Exception(f"Failed to check remote path type: {resp.text}")
330
+
331
+ path_info = resp.json()
332
+
333
+ if path_info['type'] == 'file':
334
+ # 如果是文件,调用 download_file
335
+ self.download_file(remote_file=remote_path, local_file=local_path, relpath=False)
336
+ elif path_info['type'] == 'dir':
337
+ # 如果是目录,调用 download_dir
338
+ self.download_dir(remote_dir=remote_path, local_dir=local_path,
339
+ verify_etag=verify_etag, relpath=False)
340
+ else:
341
+ raise ValueError(f"Unknown path type: {path_info['type']}")