pyxllib 0.3.197__py3-none-any.whl → 0.3.200__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. pyxllib/__init__.py +21 -21
  2. pyxllib/algo/__init__.py +8 -8
  3. pyxllib/algo/disjoint.py +54 -54
  4. pyxllib/algo/geo.py +541 -541
  5. pyxllib/algo/intervals.py +964 -964
  6. pyxllib/algo/matcher.py +389 -389
  7. pyxllib/algo/newbie.py +166 -166
  8. pyxllib/algo/pupil.py +629 -629
  9. pyxllib/algo/shapelylib.py +67 -67
  10. pyxllib/algo/specialist.py +241 -241
  11. pyxllib/algo/stat.py +494 -494
  12. pyxllib/algo/treelib.py +149 -149
  13. pyxllib/algo/unitlib.py +66 -66
  14. pyxllib/autogui/__init__.py +5 -5
  15. pyxllib/autogui/activewin.py +246 -246
  16. pyxllib/autogui/all.py +9 -9
  17. pyxllib/autogui/autogui.py +852 -852
  18. pyxllib/autogui/uiautolib.py +362 -362
  19. pyxllib/autogui/virtualkey.py +102 -102
  20. pyxllib/autogui/wechat.py +827 -827
  21. pyxllib/autogui/wechat_msg.py +421 -421
  22. pyxllib/autogui/wxautolib.py +84 -84
  23. pyxllib/cv/__init__.py +5 -5
  24. pyxllib/cv/expert.py +267 -267
  25. pyxllib/cv/imfile.py +159 -159
  26. pyxllib/cv/imhash.py +39 -39
  27. pyxllib/cv/pupil.py +9 -9
  28. pyxllib/cv/rgbfmt.py +1525 -1525
  29. pyxllib/cv/slidercaptcha.py +137 -137
  30. pyxllib/cv/trackbartools.py +251 -251
  31. pyxllib/cv/xlcvlib.py +1040 -1040
  32. pyxllib/cv/xlpillib.py +423 -423
  33. pyxllib/data/echarts.py +240 -240
  34. pyxllib/data/jsonlib.py +89 -89
  35. pyxllib/data/oss.py +72 -72
  36. pyxllib/data/pglib.py +1127 -1127
  37. pyxllib/data/sqlite.py +568 -568
  38. pyxllib/data/sqllib.py +297 -297
  39. pyxllib/ext/JLineViewer.py +505 -505
  40. pyxllib/ext/__init__.py +6 -6
  41. pyxllib/ext/demolib.py +246 -246
  42. pyxllib/ext/drissionlib.py +277 -277
  43. pyxllib/ext/kq5034lib.py +12 -12
  44. pyxllib/ext/old.py +663 -663
  45. pyxllib/ext/qt.py +449 -449
  46. pyxllib/ext/robustprocfile.py +497 -497
  47. pyxllib/ext/seleniumlib.py +76 -76
  48. pyxllib/ext/tk.py +173 -173
  49. pyxllib/ext/unixlib.py +827 -827
  50. pyxllib/ext/utools.py +351 -351
  51. pyxllib/ext/webhook.py +124 -119
  52. pyxllib/ext/win32lib.py +40 -40
  53. pyxllib/ext/wjxlib.py +88 -88
  54. pyxllib/ext/wpsapi.py +124 -124
  55. pyxllib/ext/xlwork.py +9 -9
  56. pyxllib/ext/yuquelib.py +1105 -1105
  57. pyxllib/file/__init__.py +17 -17
  58. pyxllib/file/docxlib.py +761 -761
  59. pyxllib/file/gitlib.py +309 -309
  60. pyxllib/file/libreoffice.py +165 -165
  61. pyxllib/file/movielib.py +148 -148
  62. pyxllib/file/newbie.py +10 -10
  63. pyxllib/file/onenotelib.py +1469 -1469
  64. pyxllib/file/packlib/__init__.py +330 -330
  65. pyxllib/file/packlib/zipfile.py +2441 -2441
  66. pyxllib/file/pdflib.py +426 -426
  67. pyxllib/file/pupil.py +185 -185
  68. pyxllib/file/specialist/__init__.py +685 -685
  69. pyxllib/file/specialist/dirlib.py +799 -799
  70. pyxllib/file/specialist/download.py +193 -193
  71. pyxllib/file/specialist/filelib.py +2829 -2829
  72. pyxllib/file/xlsxlib.py +3131 -3131
  73. pyxllib/file/xlsyncfile.py +341 -341
  74. pyxllib/prog/__init__.py +5 -5
  75. pyxllib/prog/cachetools.py +64 -64
  76. pyxllib/prog/deprecatedlib.py +233 -233
  77. pyxllib/prog/filelock.py +42 -42
  78. pyxllib/prog/ipyexec.py +253 -253
  79. pyxllib/prog/multiprogs.py +940 -940
  80. pyxllib/prog/newbie.py +451 -451
  81. pyxllib/prog/pupil.py +1197 -1197
  82. pyxllib/prog/sitepackages.py +33 -33
  83. pyxllib/prog/specialist/__init__.py +391 -391
  84. pyxllib/prog/specialist/bc.py +203 -203
  85. pyxllib/prog/specialist/browser.py +497 -497
  86. pyxllib/prog/specialist/common.py +347 -347
  87. pyxllib/prog/specialist/datetime.py +198 -198
  88. pyxllib/prog/specialist/tictoc.py +240 -240
  89. pyxllib/prog/specialist/xllog.py +180 -180
  90. pyxllib/prog/xlosenv.py +108 -108
  91. pyxllib/stdlib/__init__.py +17 -17
  92. pyxllib/stdlib/tablepyxl/__init__.py +10 -10
  93. pyxllib/stdlib/tablepyxl/style.py +303 -303
  94. pyxllib/stdlib/tablepyxl/tablepyxl.py +130 -130
  95. pyxllib/text/__init__.py +8 -8
  96. pyxllib/text/ahocorasick.py +39 -39
  97. pyxllib/text/airscript.js +744 -744
  98. pyxllib/text/charclasslib.py +121 -121
  99. pyxllib/text/jiebalib.py +267 -267
  100. pyxllib/text/jinjalib.py +32 -32
  101. pyxllib/text/jsa_ai_prompt.md +271 -271
  102. pyxllib/text/jscode.py +922 -922
  103. pyxllib/text/latex/__init__.py +158 -158
  104. pyxllib/text/levenshtein.py +303 -303
  105. pyxllib/text/nestenv.py +1215 -1215
  106. pyxllib/text/newbie.py +300 -300
  107. pyxllib/text/pupil/__init__.py +8 -8
  108. pyxllib/text/pupil/common.py +1121 -1121
  109. pyxllib/text/pupil/xlalign.py +326 -326
  110. pyxllib/text/pycode.py +47 -47
  111. pyxllib/text/specialist/__init__.py +8 -8
  112. pyxllib/text/specialist/common.py +112 -112
  113. pyxllib/text/specialist/ptag.py +186 -186
  114. pyxllib/text/spellchecker.py +172 -172
  115. pyxllib/text/templates/echart_base.html +10 -10
  116. pyxllib/text/templates/highlight_code.html +16 -16
  117. pyxllib/text/templates/latex_editor.html +102 -102
  118. pyxllib/text/vbacode.py +17 -17
  119. pyxllib/text/xmllib.py +747 -747
  120. pyxllib/xl.py +42 -39
  121. pyxllib/xlcv.py +17 -17
  122. {pyxllib-0.3.197.dist-info → pyxllib-0.3.200.dist-info}/METADATA +1 -1
  123. pyxllib-0.3.200.dist-info/RECORD +126 -0
  124. {pyxllib-0.3.197.dist-info → pyxllib-0.3.200.dist-info}/licenses/LICENSE +190 -190
  125. pyxllib-0.3.197.dist-info/RECORD +0 -126
  126. {pyxllib-0.3.197.dist-info → pyxllib-0.3.200.dist-info}/WHEEL +0 -0
@@ -1,341 +1,341 @@
1
- #!/usr/bin/env python3
2
- # -*- coding: utf-8 -*-
3
- # @Author : 陈坤泽
4
- # @Email : 877362867@qq.com
5
- # @Date : 2024/10/12
6
-
7
- """
8
- 我们自己的一套利用后端服务进行文件同步的工具
9
- """
10
-
11
- import os
12
- import time
13
-
14
- from loguru import logger
15
- from tqdm import tqdm
16
- import requests
17
-
18
- from pyxllib.file.specialist import XlPath, GetEtag
19
- from pyxllib.prog.pupil import format_exception
20
-
21
-
22
- class SyncFileClient:
23
- """
24
-
25
- 注意:该类填写的路径,一律是相对local_root、remote_root的路径,不是当前实际工作目录下的路径!
26
- 可以通过把local_host设为空的来实现一些特殊精细化的操作
27
- 但一般推荐还是都把local_root、remote_root设置上
28
- """
29
-
30
- def __1_basic(self):
31
- pass
32
-
33
- def __init__(self, host, token=None):
34
- """
35
- :param host: 可以只写主机名
36
- """
37
- token = token or os.getenv("XL_COMMON_SERVER_TOKEN")
38
- self.headers = {
39
- 'Authorization': f'Bearer {token}',
40
- }
41
- self.local_root = None
42
- self.remote_root = None
43
- self.host = self.link_host(host)
44
-
45
- def link_host(self, hostname):
46
- from requests.exceptions import Timeout
47
-
48
- host = f'{os.getenv("MAIN_WEBSITE")}/{hostname}'
49
- resp = requests.get(f'{host}/common/get_local_server', headers=self.headers)
50
- ip = resp.json()['ip']
51
-
52
- if ip:
53
- try:
54
- resp = requests.get(f'{ip}/healthz', timeout=5)
55
- if resp.status_code == 200 and resp.json()['status'] == 'ok':
56
- host = ip
57
- except Timeout:
58
- pass
59
- except Exception as e:
60
- raise e
61
-
62
- remote_root = requests.get(f'{host}/common/get_wkdir', headers=self.headers).json()['wkdir']
63
- self.remote_root = XlPath(remote_root)
64
-
65
- return host
66
-
67
- def get_abs_local_path(self, local_path, remote_path=None):
68
- # 1 如果需要,从远程路径推测本地路径
69
- if local_path is None and remote_path is not None:
70
- local_path = remote_path
71
-
72
- # 2 本地存储的文件路径
73
- if self.local_root:
74
- return self.local_root / local_path
75
- else:
76
- return XlPath(local_path)
77
-
78
- def get_abs_remote_path(self, remote_path, local_path=None):
79
- # 1 如果需要,从本地路径推测远程路径
80
- if remote_path is None and local_path is not None:
81
- remote_path = local_path
82
-
83
- # 2 远程存储的文件路径
84
- if self.remote_root:
85
- return self.remote_root / remote_path
86
- else:
87
- return XlPath(remote_path)
88
-
89
- def __2_sync(self):
90
- pass
91
-
92
- def upload_file(self, local_file=None, remote_file=None, relpath=True):
93
- """
94
- :param str remote_file: 要存储的远程文件路径,默认可以不指定,自动通过local_file存储到对称位置
95
-
96
- 使用示例:
97
- sfc = SyncFileClient('codepc_mi15')
98
- sfc.upload_file('abc/README.md')
99
- sfc.upload_file('abc/README.md', 'bcd/1.md')
100
- """
101
- file = [local_file]
102
- if remote_file:
103
- file.append(remote_file)
104
- return self.upload_files([file], relpath=relpath)
105
-
106
- def upload_files(self, files, relpath=True):
107
- """
108
- :param list[local_file<, remote_path>] files:
109
- remote_path可选,未写的时候会自动填充
110
- :return:
111
-
112
- todo 加进度条?不过问了下gpt,这个实现好像有点复杂,先不折腾了。本来其实就推荐单个单个文件上传,在外部进行进度展示管控。
113
-
114
- 使用示例:
115
- sfc = SyncFileClient('codepc_mi15')
116
- print(sfc.upload_files(['data/d231120禅宗1期4阶.xlsx', 'data/d231120禅宗1期4阶2.xlsx']))
117
- """
118
- # 1 映射出目标路径
119
- files2 = []
120
- for file in files:
121
- if not isinstance(file, (list, tuple)):
122
- local_file, remote_file = file, None
123
- else:
124
- local_file, remote_file = file
125
-
126
- if relpath:
127
- local_file = self.get_abs_local_path(local_file, remote_file)
128
- remote_file = self.get_abs_remote_path(remote_file, local_file)
129
-
130
- files2.append([local_file, remote_file.as_posix()])
131
-
132
- files3 = [('files', (remote_file, open(local_file, 'rb'))) for local_file, remote_file in files2]
133
-
134
- # 4 发送请求
135
- resp = requests.post(f'{self.host}/common/upload_files',
136
- headers=self.headers,
137
- files=files3)
138
- return resp.json()
139
-
140
- def download_file(self, remote_file=None, local_file=None, relpath=True):
141
- """
142
- :param str remote_file: 要下载的远程文件路径,相对get_wkdir下的路径
143
- :param str local_file: 下载到本地的文件路径,默认可以不指定,自动通过remote_file存储到对称位置
144
-
145
- todo 加进度条?方便超大文件下载查看进度。
146
-
147
- 使用示例:
148
- sfc = SyncFileClient('http://yourserver.com', headers={"Authorization": "Bearer your_token"})
149
- sfc.download_file('.vscode/launch.json', 'launch.json')
150
- """
151
- if relpath:
152
- local_file = self.get_abs_local_path(local_file, remote_file)
153
- remote_file = self.get_abs_remote_path(remote_file, local_file)
154
-
155
- data = {'file': remote_file.as_posix()}
156
- # 使用 stream=True 开启流式处理
157
- with requests.post(f'{self.host}/common/download_file',
158
- headers=self.headers, json=data, stream=True) as resp:
159
- if resp.status_code == 200:
160
- # 以二进制方式写入文件
161
- with open(local_file, 'wb') as f:
162
- # 分块写入文件,每次 1MB (1024 * 1024 bytes)
163
- for chunk in resp.iter_content(chunk_size=1024 * 1024):
164
- if chunk: # 忽略keep-alive的空块
165
- f.write(chunk)
166
- return local_file
167
- else:
168
- # 处理其他状态码,抛出异常或进行错误处理
169
- print(f"Error: Unable to download file, status code {resp.status_code}")
170
- return None
171
-
172
- def verify_etag(self, local_file=None, remote_file=None, relpath=True):
173
- """
174
- :param relpath: 是否使用的是相对路径
175
- 当不使用relpath的时候,注意local_file、remote_file都是手动必填的XlPath类型字段
176
- :return:
177
- """
178
- if relpath:
179
- local_file = self.get_abs_local_path(local_file, remote_file)
180
- remote_file = self.get_abs_remote_path(remote_file, local_file)
181
- if not local_file.is_file():
182
- return {'match': False}
183
- local_etag = GetEtag.from_file(local_file)
184
- json_data = {'file': remote_file.as_posix(), 'etag': local_etag}
185
- resp = requests.post(f'{self.host}/common/check_file_etag',
186
- headers=self.headers,
187
- json=json_data)
188
- return resp.json()
189
-
190
- def upload_dir(self,
191
- local_dir=None,
192
- remote_dir=None,
193
- *,
194
- delete_local_file=False,
195
- verify_etag=False,
196
- relpath=True):
197
- """
198
- :param local_dir: 本地目录
199
- :param remote_dir: 远程目录
200
- :param delete_local_file: 每个文件上传成功后,是否删除本地文件
201
- 默认不删除,一般不删除是配合etag校验使用的
202
- :param verify_etag: 上传前是否校验etag,如果服务器上对应位置已经有对应etag的文件,其实就不用上传了
203
-
204
- 该方法已修改以支持上传子目录和空目录
205
- """
206
- if relpath:
207
- # D:\home\chenkunze\data\temp2,举个具体的例子案例值,供参考理解
208
- local_dir = self.get_abs_local_path(local_dir, remote_dir)
209
- # /home/chenkunze/data/temp2
210
- remote_dir = self.get_abs_remote_path(remote_dir, local_dir)
211
-
212
- for root, dirs, files in os.walk(local_dir):
213
- # root拿到的是这样的绝对路径:D:\home\chenkunze\data\temp2\数值问题
214
- # '数值问题'
215
- subdir = XlPath(root).relpath(local_dir)
216
- if not files and not dirs:
217
- # 如果该目录为空,则在远程服务器上创建空目录
218
- self.create_remote_dir(remote_dir / subdir, relpath=False)
219
-
220
- for file in tqdm(files, desc=f'上传目录:{root}'):
221
- local_path = local_dir / subdir / file
222
- remote_path = remote_dir / subdir / file
223
- if verify_etag and self.verify_etag(local_path, remote_path, relpath=False):
224
- pass # 如果etag校验通过,跳过上传
225
- else:
226
- self.upload_file(local_path, remote_path, relpath=False)
227
-
228
- if delete_local_file:
229
- os.remove(local_path)
230
-
231
- def create_remote_dir(self, remote_dir, relpath=True):
232
- """
233
- 创建远程目录的辅助方法
234
- """
235
- if relpath:
236
- remote_dir = self.get_abs_remote_path(remote_dir)
237
- json_data = {'dir': remote_dir.as_posix()}
238
- resp = requests.post(f'{self.host}/common/create_dir',
239
- headers=self.headers,
240
- json=json_data)
241
- return resp.json()
242
-
243
- def download_dir(self, remote_dir=None, local_dir=None, *, verify_etag=False, relpath=True):
244
- """
245
- 下载远程目录到本地
246
- :param remote_dir: 远程目录路径,相对路径或绝对路径都支持
247
- :param local_dir: 本地保存目录,默认会自动映射到对应的远程位置
248
- :param verify_etag: 是否进行etag校验,跳过已经在本地存在并且未修改的文件
249
- :return:
250
-
251
- 示例:
252
- sfc = SyncFileClient('codepc_mi15')
253
- sfc.download_dir('data/temp2')
254
- """
255
- # 1. 获取绝对路径
256
- if relpath:
257
- local_dir = self.get_abs_local_path(local_dir, remote_dir)
258
- remote_dir = self.get_abs_remote_path(remote_dir, local_dir)
259
-
260
- # 2. 从服务器获取远程目录的文件和子目录结构
261
- json_data = {'dir': remote_dir.as_posix()}
262
- resp = requests.post(f'{self.host}/common/list_dir', headers=self.headers, json=json_data)
263
- if resp.status_code != 200:
264
- raise Exception(f"Failed to list remote directory: {resp.text}")
265
-
266
- dir_structure = resp.json() # 期望返回格式为 {'dirs': [...], 'files': [...]}
267
-
268
- # 3. 创建本地目录
269
- if not local_dir.exists():
270
- local_dir.mkdir(parents=True)
271
-
272
- # 4. 先处理子目录
273
- for subdir in dir_structure.get('dirs', []):
274
- local_subdir = local_dir / subdir
275
- if not local_subdir.exists():
276
- local_subdir.mkdir(parents=True)
277
-
278
- # 递归下载子目录中的文件
279
- for i in range(100): # 开发机senseserver3好像不稳定,加了重试机制
280
- try:
281
- self.download_dir(remote_dir / subdir, local_dir / subdir, verify_etag=verify_etag)
282
- break
283
- except requests.exceptions.RequestException as e:
284
- logger.warning(format_exception(e, 3))
285
- time.sleep(60)
286
- continue
287
- else:
288
- raise requests.exceptions.RequestException
289
-
290
- # 5. 下载文件
291
- for file in tqdm(dir_structure.get('files', []), desc=f'下载目录:{remote_dir}'):
292
- remote_file = remote_dir / file
293
- local_file = local_dir / file
294
-
295
- for i in range(100):
296
- try:
297
- # 校验etag,如果verify_etag为True并且校验成功,则跳过下载
298
- if verify_etag and self.verify_etag(local_file, remote_file, relpath=False)['match']:
299
- break
300
-
301
- # 下载文件
302
- logger.info(f'下载文件中:{local_file}')
303
- self.download_file(remote_file, local_file, relpath=False)
304
- break
305
- except requests.exceptions.RequestException:
306
- time.sleep(60)
307
- continue
308
- else:
309
- raise requests.exceptions.RequestException
310
-
311
- def download_path(self, remote_path=None, local_path=None, *, verify_etag=False, relpath=True):
312
- """
313
- 下载远程文件或目录到本地
314
- :param remote_path: 远程路径,可以是文件或者目录
315
- :param local_path: 本地保存路径,默认会自动映射到对应的远程位置
316
- :param verify_etag: 是否进行etag校验,跳过已经在本地存在并且未修改的文件
317
- :return:
318
- """
319
- if relpath:
320
- # 获取远程路径的绝对路径
321
- remote_path = self.get_abs_remote_path(remote_path, local_path)
322
- local_path = self.get_abs_local_path(local_path, remote_path)
323
-
324
- # 判断是文件还是目录
325
- json_data = {'path': remote_path.as_posix()}
326
- resp = requests.post(f'{self.host}/common/check_path_type', headers=self.headers, json=json_data)
327
-
328
- if resp.status_code != 200:
329
- raise Exception(f"Failed to check remote path type: {resp.text}")
330
-
331
- path_info = resp.json()
332
-
333
- if path_info['type'] == 'file':
334
- # 如果是文件,调用 download_file
335
- self.download_file(remote_file=remote_path, local_file=local_path, relpath=False)
336
- elif path_info['type'] == 'dir':
337
- # 如果是目录,调用 download_dir
338
- self.download_dir(remote_dir=remote_path, local_dir=local_path,
339
- verify_etag=verify_etag, relpath=False)
340
- else:
341
- raise ValueError(f"Unknown path type: {path_info['type']}")
1
+ #!/usr/bin/env python3
2
+ # -*- coding: utf-8 -*-
3
+ # @Author : 陈坤泽
4
+ # @Email : 877362867@qq.com
5
+ # @Date : 2024/10/12
6
+
7
+ """
8
+ 我们自己的一套利用后端服务进行文件同步的工具
9
+ """
10
+
11
+ import os
12
+ import time
13
+
14
+ from loguru import logger
15
+ from tqdm import tqdm
16
+ import requests
17
+
18
+ from pyxllib.file.specialist import XlPath, GetEtag
19
+ from pyxllib.prog.pupil import format_exception
20
+
21
+
22
+ class SyncFileClient:
23
+ """
24
+
25
+ 注意:该类填写的路径,一律是相对local_root、remote_root的路径,不是当前实际工作目录下的路径!
26
+ 可以通过把local_host设为空的来实现一些特殊精细化的操作
27
+ 但一般推荐还是都把local_root、remote_root设置上
28
+ """
29
+
30
+ def __1_basic(self):
31
+ pass
32
+
33
+ def __init__(self, host, token=None):
34
+ """
35
+ :param host: 可以只写主机名
36
+ """
37
+ token = token or os.getenv("XL_COMMON_SERVER_TOKEN")
38
+ self.headers = {
39
+ 'Authorization': f'Bearer {token}',
40
+ }
41
+ self.local_root = None
42
+ self.remote_root = None
43
+ self.host = self.link_host(host)
44
+
45
+ def link_host(self, hostname):
46
+ from requests.exceptions import Timeout
47
+
48
+ host = f'{os.getenv("MAIN_WEBSITE")}/{hostname}'
49
+ resp = requests.get(f'{host}/common/get_local_server', headers=self.headers)
50
+ ip = resp.json()['ip']
51
+
52
+ if ip:
53
+ try:
54
+ resp = requests.get(f'{ip}/healthz', timeout=5)
55
+ if resp.status_code == 200 and resp.json()['status'] == 'ok':
56
+ host = ip
57
+ except Timeout:
58
+ pass
59
+ except Exception as e:
60
+ raise e
61
+
62
+ remote_root = requests.get(f'{host}/common/get_wkdir', headers=self.headers).json()['wkdir']
63
+ self.remote_root = XlPath(remote_root)
64
+
65
+ return host
66
+
67
+ def get_abs_local_path(self, local_path, remote_path=None):
68
+ # 1 如果需要,从远程路径推测本地路径
69
+ if local_path is None and remote_path is not None:
70
+ local_path = remote_path
71
+
72
+ # 2 本地存储的文件路径
73
+ if self.local_root:
74
+ return self.local_root / local_path
75
+ else:
76
+ return XlPath(local_path)
77
+
78
+ def get_abs_remote_path(self, remote_path, local_path=None):
79
+ # 1 如果需要,从本地路径推测远程路径
80
+ if remote_path is None and local_path is not None:
81
+ remote_path = local_path
82
+
83
+ # 2 远程存储的文件路径
84
+ if self.remote_root:
85
+ return self.remote_root / remote_path
86
+ else:
87
+ return XlPath(remote_path)
88
+
89
+ def __2_sync(self):
90
+ pass
91
+
92
+ def upload_file(self, local_file=None, remote_file=None, relpath=True):
93
+ """
94
+ :param str remote_file: 要存储的远程文件路径,默认可以不指定,自动通过local_file存储到对称位置
95
+
96
+ 使用示例:
97
+ sfc = SyncFileClient('codepc_mi15')
98
+ sfc.upload_file('abc/README.md')
99
+ sfc.upload_file('abc/README.md', 'bcd/1.md')
100
+ """
101
+ file = [local_file]
102
+ if remote_file:
103
+ file.append(remote_file)
104
+ return self.upload_files([file], relpath=relpath)
105
+
106
+ def upload_files(self, files, relpath=True):
107
+ """
108
+ :param list[local_file<, remote_path>] files:
109
+ remote_path可选,未写的时候会自动填充
110
+ :return:
111
+
112
+ todo 加进度条?不过问了下gpt,这个实现好像有点复杂,先不折腾了。本来其实就推荐单个单个文件上传,在外部进行进度展示管控。
113
+
114
+ 使用示例:
115
+ sfc = SyncFileClient('codepc_mi15')
116
+ print(sfc.upload_files(['data/d231120禅宗1期4阶.xlsx', 'data/d231120禅宗1期4阶2.xlsx']))
117
+ """
118
+ # 1 映射出目标路径
119
+ files2 = []
120
+ for file in files:
121
+ if not isinstance(file, (list, tuple)):
122
+ local_file, remote_file = file, None
123
+ else:
124
+ local_file, remote_file = file
125
+
126
+ if relpath:
127
+ local_file = self.get_abs_local_path(local_file, remote_file)
128
+ remote_file = self.get_abs_remote_path(remote_file, local_file)
129
+
130
+ files2.append([local_file, remote_file.as_posix()])
131
+
132
+ files3 = [('files', (remote_file, open(local_file, 'rb'))) for local_file, remote_file in files2]
133
+
134
+ # 4 发送请求
135
+ resp = requests.post(f'{self.host}/common/upload_files',
136
+ headers=self.headers,
137
+ files=files3)
138
+ return resp.json()
139
+
140
+ def download_file(self, remote_file=None, local_file=None, relpath=True):
141
+ """
142
+ :param str remote_file: 要下载的远程文件路径,相对get_wkdir下的路径
143
+ :param str local_file: 下载到本地的文件路径,默认可以不指定,自动通过remote_file存储到对称位置
144
+
145
+ todo 加进度条?方便超大文件下载查看进度。
146
+
147
+ 使用示例:
148
+ sfc = SyncFileClient('http://yourserver.com', headers={"Authorization": "Bearer your_token"})
149
+ sfc.download_file('.vscode/launch.json', 'launch.json')
150
+ """
151
+ if relpath:
152
+ local_file = self.get_abs_local_path(local_file, remote_file)
153
+ remote_file = self.get_abs_remote_path(remote_file, local_file)
154
+
155
+ data = {'file': remote_file.as_posix()}
156
+ # 使用 stream=True 开启流式处理
157
+ with requests.post(f'{self.host}/common/download_file',
158
+ headers=self.headers, json=data, stream=True) as resp:
159
+ if resp.status_code == 200:
160
+ # 以二进制方式写入文件
161
+ with open(local_file, 'wb') as f:
162
+ # 分块写入文件,每次 1MB (1024 * 1024 bytes)
163
+ for chunk in resp.iter_content(chunk_size=1024 * 1024):
164
+ if chunk: # 忽略keep-alive的空块
165
+ f.write(chunk)
166
+ return local_file
167
+ else:
168
+ # 处理其他状态码,抛出异常或进行错误处理
169
+ print(f"Error: Unable to download file, status code {resp.status_code}")
170
+ return None
171
+
172
+ def verify_etag(self, local_file=None, remote_file=None, relpath=True):
173
+ """
174
+ :param relpath: 是否使用的是相对路径
175
+ 当不使用relpath的时候,注意local_file、remote_file都是手动必填的XlPath类型字段
176
+ :return:
177
+ """
178
+ if relpath:
179
+ local_file = self.get_abs_local_path(local_file, remote_file)
180
+ remote_file = self.get_abs_remote_path(remote_file, local_file)
181
+ if not local_file.is_file():
182
+ return {'match': False}
183
+ local_etag = GetEtag.from_file(local_file)
184
+ json_data = {'file': remote_file.as_posix(), 'etag': local_etag}
185
+ resp = requests.post(f'{self.host}/common/check_file_etag',
186
+ headers=self.headers,
187
+ json=json_data)
188
+ return resp.json()
189
+
190
+ def upload_dir(self,
191
+ local_dir=None,
192
+ remote_dir=None,
193
+ *,
194
+ delete_local_file=False,
195
+ verify_etag=False,
196
+ relpath=True):
197
+ """
198
+ :param local_dir: 本地目录
199
+ :param remote_dir: 远程目录
200
+ :param delete_local_file: 每个文件上传成功后,是否删除本地文件
201
+ 默认不删除,一般不删除是配合etag校验使用的
202
+ :param verify_etag: 上传前是否校验etag,如果服务器上对应位置已经有对应etag的文件,其实就不用上传了
203
+
204
+ 该方法已修改以支持上传子目录和空目录
205
+ """
206
+ if relpath:
207
+ # D:\home\chenkunze\data\temp2,举个具体的例子案例值,供参考理解
208
+ local_dir = self.get_abs_local_path(local_dir, remote_dir)
209
+ # /home/chenkunze/data/temp2
210
+ remote_dir = self.get_abs_remote_path(remote_dir, local_dir)
211
+
212
+ for root, dirs, files in os.walk(local_dir):
213
+ # root拿到的是这样的绝对路径:D:\home\chenkunze\data\temp2\数值问题
214
+ # '数值问题'
215
+ subdir = XlPath(root).relpath(local_dir)
216
+ if not files and not dirs:
217
+ # 如果该目录为空,则在远程服务器上创建空目录
218
+ self.create_remote_dir(remote_dir / subdir, relpath=False)
219
+
220
+ for file in tqdm(files, desc=f'上传目录:{root}'):
221
+ local_path = local_dir / subdir / file
222
+ remote_path = remote_dir / subdir / file
223
+ if verify_etag and self.verify_etag(local_path, remote_path, relpath=False):
224
+ pass # 如果etag校验通过,跳过上传
225
+ else:
226
+ self.upload_file(local_path, remote_path, relpath=False)
227
+
228
+ if delete_local_file:
229
+ os.remove(local_path)
230
+
231
+ def create_remote_dir(self, remote_dir, relpath=True):
232
+ """
233
+ 创建远程目录的辅助方法
234
+ """
235
+ if relpath:
236
+ remote_dir = self.get_abs_remote_path(remote_dir)
237
+ json_data = {'dir': remote_dir.as_posix()}
238
+ resp = requests.post(f'{self.host}/common/create_dir',
239
+ headers=self.headers,
240
+ json=json_data)
241
+ return resp.json()
242
+
243
+ def download_dir(self, remote_dir=None, local_dir=None, *, verify_etag=False, relpath=True):
244
+ """
245
+ 下载远程目录到本地
246
+ :param remote_dir: 远程目录路径,相对路径或绝对路径都支持
247
+ :param local_dir: 本地保存目录,默认会自动映射到对应的远程位置
248
+ :param verify_etag: 是否进行etag校验,跳过已经在本地存在并且未修改的文件
249
+ :return:
250
+
251
+ 示例:
252
+ sfc = SyncFileClient('codepc_mi15')
253
+ sfc.download_dir('data/temp2')
254
+ """
255
+ # 1. 获取绝对路径
256
+ if relpath:
257
+ local_dir = self.get_abs_local_path(local_dir, remote_dir)
258
+ remote_dir = self.get_abs_remote_path(remote_dir, local_dir)
259
+
260
+ # 2. 从服务器获取远程目录的文件和子目录结构
261
+ json_data = {'dir': remote_dir.as_posix()}
262
+ resp = requests.post(f'{self.host}/common/list_dir', headers=self.headers, json=json_data)
263
+ if resp.status_code != 200:
264
+ raise Exception(f"Failed to list remote directory: {resp.text}")
265
+
266
+ dir_structure = resp.json() # 期望返回格式为 {'dirs': [...], 'files': [...]}
267
+
268
+ # 3. 创建本地目录
269
+ if not local_dir.exists():
270
+ local_dir.mkdir(parents=True)
271
+
272
+ # 4. 先处理子目录
273
+ for subdir in dir_structure.get('dirs', []):
274
+ local_subdir = local_dir / subdir
275
+ if not local_subdir.exists():
276
+ local_subdir.mkdir(parents=True)
277
+
278
+ # 递归下载子目录中的文件
279
+ for i in range(100): # 开发机senseserver3好像不稳定,加了重试机制
280
+ try:
281
+ self.download_dir(remote_dir / subdir, local_dir / subdir, verify_etag=verify_etag)
282
+ break
283
+ except requests.exceptions.RequestException as e:
284
+ logger.warning(format_exception(e, 3))
285
+ time.sleep(60)
286
+ continue
287
+ else:
288
+ raise requests.exceptions.RequestException
289
+
290
+ # 5. 下载文件
291
+ for file in tqdm(dir_structure.get('files', []), desc=f'下载目录:{remote_dir}'):
292
+ remote_file = remote_dir / file
293
+ local_file = local_dir / file
294
+
295
+ for i in range(100):
296
+ try:
297
+ # 校验etag,如果verify_etag为True并且校验成功,则跳过下载
298
+ if verify_etag and self.verify_etag(local_file, remote_file, relpath=False)['match']:
299
+ break
300
+
301
+ # 下载文件
302
+ logger.info(f'下载文件中:{local_file}')
303
+ self.download_file(remote_file, local_file, relpath=False)
304
+ break
305
+ except requests.exceptions.RequestException:
306
+ time.sleep(60)
307
+ continue
308
+ else:
309
+ raise requests.exceptions.RequestException
310
+
311
+ def download_path(self, remote_path=None, local_path=None, *, verify_etag=False, relpath=True):
312
+ """
313
+ 下载远程文件或目录到本地
314
+ :param remote_path: 远程路径,可以是文件或者目录
315
+ :param local_path: 本地保存路径,默认会自动映射到对应的远程位置
316
+ :param verify_etag: 是否进行etag校验,跳过已经在本地存在并且未修改的文件
317
+ :return:
318
+ """
319
+ if relpath:
320
+ # 获取远程路径的绝对路径
321
+ remote_path = self.get_abs_remote_path(remote_path, local_path)
322
+ local_path = self.get_abs_local_path(local_path, remote_path)
323
+
324
+ # 判断是文件还是目录
325
+ json_data = {'path': remote_path.as_posix()}
326
+ resp = requests.post(f'{self.host}/common/check_path_type', headers=self.headers, json=json_data)
327
+
328
+ if resp.status_code != 200:
329
+ raise Exception(f"Failed to check remote path type: {resp.text}")
330
+
331
+ path_info = resp.json()
332
+
333
+ if path_info['type'] == 'file':
334
+ # 如果是文件,调用 download_file
335
+ self.download_file(remote_file=remote_path, local_file=local_path, relpath=False)
336
+ elif path_info['type'] == 'dir':
337
+ # 如果是目录,调用 download_dir
338
+ self.download_dir(remote_dir=remote_path, local_dir=local_path,
339
+ verify_etag=verify_etag, relpath=False)
340
+ else:
341
+ raise ValueError(f"Unknown path type: {path_info['type']}")