pyxllib 0.3.96__py3-none-any.whl → 0.3.197__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (306) hide show
  1. pyxllib/algo/geo.py +12 -0
  2. pyxllib/algo/intervals.py +1 -1
  3. pyxllib/algo/matcher.py +78 -0
  4. pyxllib/algo/pupil.py +187 -19
  5. pyxllib/algo/specialist.py +2 -1
  6. pyxllib/algo/stat.py +38 -2
  7. {pyxlpr → pyxllib/autogui}/__init__.py +1 -1
  8. pyxllib/autogui/activewin.py +246 -0
  9. pyxllib/autogui/all.py +9 -0
  10. pyxllib/{ext/autogui → autogui}/autogui.py +40 -11
  11. pyxllib/autogui/uiautolib.py +362 -0
  12. pyxllib/autogui/wechat.py +827 -0
  13. pyxllib/autogui/wechat_msg.py +421 -0
  14. pyxllib/autogui/wxautolib.py +84 -0
  15. pyxllib/cv/slidercaptcha.py +137 -0
  16. pyxllib/data/echarts.py +123 -12
  17. pyxllib/data/jsonlib.py +89 -0
  18. pyxllib/data/pglib.py +514 -30
  19. pyxllib/data/sqlite.py +231 -4
  20. pyxllib/ext/JLineViewer.py +14 -1
  21. pyxllib/ext/drissionlib.py +277 -0
  22. pyxllib/ext/kq5034lib.py +0 -1594
  23. pyxllib/ext/robustprocfile.py +497 -0
  24. pyxllib/ext/unixlib.py +6 -5
  25. pyxllib/ext/utools.py +108 -95
  26. pyxllib/ext/webhook.py +32 -14
  27. pyxllib/ext/wjxlib.py +88 -0
  28. pyxllib/ext/wpsapi.py +124 -0
  29. pyxllib/ext/xlwork.py +9 -0
  30. pyxllib/ext/yuquelib.py +1003 -71
  31. pyxllib/file/docxlib.py +1 -1
  32. pyxllib/file/libreoffice.py +165 -0
  33. pyxllib/file/movielib.py +9 -0
  34. pyxllib/file/packlib/__init__.py +112 -75
  35. pyxllib/file/pdflib.py +1 -1
  36. pyxllib/file/pupil.py +1 -1
  37. pyxllib/file/specialist/dirlib.py +1 -1
  38. pyxllib/file/specialist/download.py +10 -3
  39. pyxllib/file/specialist/filelib.py +266 -55
  40. pyxllib/file/xlsxlib.py +205 -50
  41. pyxllib/file/xlsyncfile.py +341 -0
  42. pyxllib/prog/cachetools.py +64 -0
  43. pyxllib/prog/filelock.py +42 -0
  44. pyxllib/prog/multiprogs.py +940 -0
  45. pyxllib/prog/newbie.py +9 -2
  46. pyxllib/prog/pupil.py +129 -60
  47. pyxllib/prog/specialist/__init__.py +176 -2
  48. pyxllib/prog/specialist/bc.py +5 -2
  49. pyxllib/prog/specialist/browser.py +11 -2
  50. pyxllib/prog/specialist/datetime.py +68 -0
  51. pyxllib/prog/specialist/tictoc.py +12 -13
  52. pyxllib/prog/specialist/xllog.py +5 -5
  53. pyxllib/prog/xlosenv.py +7 -0
  54. pyxllib/text/airscript.js +744 -0
  55. pyxllib/text/charclasslib.py +17 -5
  56. pyxllib/text/jiebalib.py +6 -3
  57. pyxllib/text/jinjalib.py +32 -0
  58. pyxllib/text/jsa_ai_prompt.md +271 -0
  59. pyxllib/text/jscode.py +159 -4
  60. pyxllib/text/nestenv.py +1 -1
  61. pyxllib/text/newbie.py +12 -0
  62. pyxllib/text/pupil/common.py +26 -0
  63. pyxllib/text/specialist/ptag.py +2 -2
  64. pyxllib/text/templates/echart_base.html +11 -0
  65. pyxllib/text/templates/highlight_code.html +17 -0
  66. pyxllib/text/templates/latex_editor.html +103 -0
  67. pyxllib/text/xmllib.py +76 -14
  68. pyxllib/xl.py +2 -1
  69. pyxllib-0.3.197.dist-info/METADATA +48 -0
  70. pyxllib-0.3.197.dist-info/RECORD +126 -0
  71. {pyxllib-0.3.96.dist-info → pyxllib-0.3.197.dist-info}/WHEEL +1 -2
  72. pyxllib/ext/autogui/__init__.py +0 -8
  73. pyxllib-0.3.96.dist-info/METADATA +0 -51
  74. pyxllib-0.3.96.dist-info/RECORD +0 -333
  75. pyxllib-0.3.96.dist-info/top_level.txt +0 -2
  76. pyxlpr/ai/__init__.py +0 -5
  77. pyxlpr/ai/clientlib.py +0 -1281
  78. pyxlpr/ai/specialist.py +0 -286
  79. pyxlpr/ai/torch_app.py +0 -172
  80. pyxlpr/ai/xlpaddle.py +0 -655
  81. pyxlpr/ai/xltorch.py +0 -705
  82. pyxlpr/data/__init__.py +0 -11
  83. pyxlpr/data/coco.py +0 -1325
  84. pyxlpr/data/datacls.py +0 -365
  85. pyxlpr/data/datasets.py +0 -200
  86. pyxlpr/data/gptlib.py +0 -1291
  87. pyxlpr/data/icdar/__init__.py +0 -96
  88. pyxlpr/data/icdar/deteval.py +0 -377
  89. pyxlpr/data/icdar/icdar2013.py +0 -341
  90. pyxlpr/data/icdar/iou.py +0 -340
  91. pyxlpr/data/icdar/rrc_evaluation_funcs_1_1.py +0 -463
  92. pyxlpr/data/imtextline.py +0 -473
  93. pyxlpr/data/labelme.py +0 -866
  94. pyxlpr/data/removeline.py +0 -179
  95. pyxlpr/data/specialist.py +0 -57
  96. pyxlpr/eval/__init__.py +0 -85
  97. pyxlpr/paddleocr.py +0 -776
  98. pyxlpr/ppocr/__init__.py +0 -15
  99. pyxlpr/ppocr/configs/rec/multi_language/generate_multi_language_configs.py +0 -226
  100. pyxlpr/ppocr/data/__init__.py +0 -135
  101. pyxlpr/ppocr/data/imaug/ColorJitter.py +0 -26
  102. pyxlpr/ppocr/data/imaug/__init__.py +0 -67
  103. pyxlpr/ppocr/data/imaug/copy_paste.py +0 -170
  104. pyxlpr/ppocr/data/imaug/east_process.py +0 -437
  105. pyxlpr/ppocr/data/imaug/gen_table_mask.py +0 -244
  106. pyxlpr/ppocr/data/imaug/iaa_augment.py +0 -114
  107. pyxlpr/ppocr/data/imaug/label_ops.py +0 -789
  108. pyxlpr/ppocr/data/imaug/make_border_map.py +0 -184
  109. pyxlpr/ppocr/data/imaug/make_pse_gt.py +0 -106
  110. pyxlpr/ppocr/data/imaug/make_shrink_map.py +0 -126
  111. pyxlpr/ppocr/data/imaug/operators.py +0 -433
  112. pyxlpr/ppocr/data/imaug/pg_process.py +0 -906
  113. pyxlpr/ppocr/data/imaug/randaugment.py +0 -143
  114. pyxlpr/ppocr/data/imaug/random_crop_data.py +0 -239
  115. pyxlpr/ppocr/data/imaug/rec_img_aug.py +0 -533
  116. pyxlpr/ppocr/data/imaug/sast_process.py +0 -777
  117. pyxlpr/ppocr/data/imaug/text_image_aug/__init__.py +0 -17
  118. pyxlpr/ppocr/data/imaug/text_image_aug/augment.py +0 -120
  119. pyxlpr/ppocr/data/imaug/text_image_aug/warp_mls.py +0 -168
  120. pyxlpr/ppocr/data/lmdb_dataset.py +0 -115
  121. pyxlpr/ppocr/data/pgnet_dataset.py +0 -104
  122. pyxlpr/ppocr/data/pubtab_dataset.py +0 -107
  123. pyxlpr/ppocr/data/simple_dataset.py +0 -372
  124. pyxlpr/ppocr/losses/__init__.py +0 -61
  125. pyxlpr/ppocr/losses/ace_loss.py +0 -52
  126. pyxlpr/ppocr/losses/basic_loss.py +0 -135
  127. pyxlpr/ppocr/losses/center_loss.py +0 -88
  128. pyxlpr/ppocr/losses/cls_loss.py +0 -30
  129. pyxlpr/ppocr/losses/combined_loss.py +0 -67
  130. pyxlpr/ppocr/losses/det_basic_loss.py +0 -208
  131. pyxlpr/ppocr/losses/det_db_loss.py +0 -80
  132. pyxlpr/ppocr/losses/det_east_loss.py +0 -63
  133. pyxlpr/ppocr/losses/det_pse_loss.py +0 -149
  134. pyxlpr/ppocr/losses/det_sast_loss.py +0 -121
  135. pyxlpr/ppocr/losses/distillation_loss.py +0 -272
  136. pyxlpr/ppocr/losses/e2e_pg_loss.py +0 -140
  137. pyxlpr/ppocr/losses/kie_sdmgr_loss.py +0 -113
  138. pyxlpr/ppocr/losses/rec_aster_loss.py +0 -99
  139. pyxlpr/ppocr/losses/rec_att_loss.py +0 -39
  140. pyxlpr/ppocr/losses/rec_ctc_loss.py +0 -44
  141. pyxlpr/ppocr/losses/rec_enhanced_ctc_loss.py +0 -70
  142. pyxlpr/ppocr/losses/rec_nrtr_loss.py +0 -30
  143. pyxlpr/ppocr/losses/rec_sar_loss.py +0 -28
  144. pyxlpr/ppocr/losses/rec_srn_loss.py +0 -47
  145. pyxlpr/ppocr/losses/table_att_loss.py +0 -109
  146. pyxlpr/ppocr/metrics/__init__.py +0 -44
  147. pyxlpr/ppocr/metrics/cls_metric.py +0 -45
  148. pyxlpr/ppocr/metrics/det_metric.py +0 -82
  149. pyxlpr/ppocr/metrics/distillation_metric.py +0 -73
  150. pyxlpr/ppocr/metrics/e2e_metric.py +0 -86
  151. pyxlpr/ppocr/metrics/eval_det_iou.py +0 -274
  152. pyxlpr/ppocr/metrics/kie_metric.py +0 -70
  153. pyxlpr/ppocr/metrics/rec_metric.py +0 -75
  154. pyxlpr/ppocr/metrics/table_metric.py +0 -50
  155. pyxlpr/ppocr/modeling/architectures/__init__.py +0 -32
  156. pyxlpr/ppocr/modeling/architectures/base_model.py +0 -88
  157. pyxlpr/ppocr/modeling/architectures/distillation_model.py +0 -60
  158. pyxlpr/ppocr/modeling/backbones/__init__.py +0 -54
  159. pyxlpr/ppocr/modeling/backbones/det_mobilenet_v3.py +0 -268
  160. pyxlpr/ppocr/modeling/backbones/det_resnet_vd.py +0 -246
  161. pyxlpr/ppocr/modeling/backbones/det_resnet_vd_sast.py +0 -285
  162. pyxlpr/ppocr/modeling/backbones/e2e_resnet_vd_pg.py +0 -265
  163. pyxlpr/ppocr/modeling/backbones/kie_unet_sdmgr.py +0 -186
  164. pyxlpr/ppocr/modeling/backbones/rec_mobilenet_v3.py +0 -138
  165. pyxlpr/ppocr/modeling/backbones/rec_mv1_enhance.py +0 -258
  166. pyxlpr/ppocr/modeling/backbones/rec_nrtr_mtb.py +0 -48
  167. pyxlpr/ppocr/modeling/backbones/rec_resnet_31.py +0 -210
  168. pyxlpr/ppocr/modeling/backbones/rec_resnet_aster.py +0 -143
  169. pyxlpr/ppocr/modeling/backbones/rec_resnet_fpn.py +0 -307
  170. pyxlpr/ppocr/modeling/backbones/rec_resnet_vd.py +0 -286
  171. pyxlpr/ppocr/modeling/heads/__init__.py +0 -54
  172. pyxlpr/ppocr/modeling/heads/cls_head.py +0 -52
  173. pyxlpr/ppocr/modeling/heads/det_db_head.py +0 -118
  174. pyxlpr/ppocr/modeling/heads/det_east_head.py +0 -121
  175. pyxlpr/ppocr/modeling/heads/det_pse_head.py +0 -37
  176. pyxlpr/ppocr/modeling/heads/det_sast_head.py +0 -128
  177. pyxlpr/ppocr/modeling/heads/e2e_pg_head.py +0 -253
  178. pyxlpr/ppocr/modeling/heads/kie_sdmgr_head.py +0 -206
  179. pyxlpr/ppocr/modeling/heads/multiheadAttention.py +0 -163
  180. pyxlpr/ppocr/modeling/heads/rec_aster_head.py +0 -393
  181. pyxlpr/ppocr/modeling/heads/rec_att_head.py +0 -202
  182. pyxlpr/ppocr/modeling/heads/rec_ctc_head.py +0 -88
  183. pyxlpr/ppocr/modeling/heads/rec_nrtr_head.py +0 -826
  184. pyxlpr/ppocr/modeling/heads/rec_sar_head.py +0 -402
  185. pyxlpr/ppocr/modeling/heads/rec_srn_head.py +0 -280
  186. pyxlpr/ppocr/modeling/heads/self_attention.py +0 -406
  187. pyxlpr/ppocr/modeling/heads/table_att_head.py +0 -246
  188. pyxlpr/ppocr/modeling/necks/__init__.py +0 -32
  189. pyxlpr/ppocr/modeling/necks/db_fpn.py +0 -111
  190. pyxlpr/ppocr/modeling/necks/east_fpn.py +0 -188
  191. pyxlpr/ppocr/modeling/necks/fpn.py +0 -138
  192. pyxlpr/ppocr/modeling/necks/pg_fpn.py +0 -314
  193. pyxlpr/ppocr/modeling/necks/rnn.py +0 -92
  194. pyxlpr/ppocr/modeling/necks/sast_fpn.py +0 -284
  195. pyxlpr/ppocr/modeling/necks/table_fpn.py +0 -110
  196. pyxlpr/ppocr/modeling/transforms/__init__.py +0 -28
  197. pyxlpr/ppocr/modeling/transforms/stn.py +0 -135
  198. pyxlpr/ppocr/modeling/transforms/tps.py +0 -308
  199. pyxlpr/ppocr/modeling/transforms/tps_spatial_transformer.py +0 -156
  200. pyxlpr/ppocr/optimizer/__init__.py +0 -61
  201. pyxlpr/ppocr/optimizer/learning_rate.py +0 -228
  202. pyxlpr/ppocr/optimizer/lr_scheduler.py +0 -49
  203. pyxlpr/ppocr/optimizer/optimizer.py +0 -160
  204. pyxlpr/ppocr/optimizer/regularizer.py +0 -52
  205. pyxlpr/ppocr/postprocess/__init__.py +0 -55
  206. pyxlpr/ppocr/postprocess/cls_postprocess.py +0 -33
  207. pyxlpr/ppocr/postprocess/db_postprocess.py +0 -234
  208. pyxlpr/ppocr/postprocess/east_postprocess.py +0 -143
  209. pyxlpr/ppocr/postprocess/locality_aware_nms.py +0 -200
  210. pyxlpr/ppocr/postprocess/pg_postprocess.py +0 -52
  211. pyxlpr/ppocr/postprocess/pse_postprocess/__init__.py +0 -15
  212. pyxlpr/ppocr/postprocess/pse_postprocess/pse/__init__.py +0 -29
  213. pyxlpr/ppocr/postprocess/pse_postprocess/pse/setup.py +0 -14
  214. pyxlpr/ppocr/postprocess/pse_postprocess/pse_postprocess.py +0 -118
  215. pyxlpr/ppocr/postprocess/rec_postprocess.py +0 -654
  216. pyxlpr/ppocr/postprocess/sast_postprocess.py +0 -355
  217. pyxlpr/ppocr/tools/__init__.py +0 -14
  218. pyxlpr/ppocr/tools/eval.py +0 -83
  219. pyxlpr/ppocr/tools/export_center.py +0 -77
  220. pyxlpr/ppocr/tools/export_model.py +0 -129
  221. pyxlpr/ppocr/tools/infer/predict_cls.py +0 -151
  222. pyxlpr/ppocr/tools/infer/predict_det.py +0 -300
  223. pyxlpr/ppocr/tools/infer/predict_e2e.py +0 -169
  224. pyxlpr/ppocr/tools/infer/predict_rec.py +0 -414
  225. pyxlpr/ppocr/tools/infer/predict_system.py +0 -204
  226. pyxlpr/ppocr/tools/infer/utility.py +0 -629
  227. pyxlpr/ppocr/tools/infer_cls.py +0 -83
  228. pyxlpr/ppocr/tools/infer_det.py +0 -134
  229. pyxlpr/ppocr/tools/infer_e2e.py +0 -122
  230. pyxlpr/ppocr/tools/infer_kie.py +0 -153
  231. pyxlpr/ppocr/tools/infer_rec.py +0 -146
  232. pyxlpr/ppocr/tools/infer_table.py +0 -107
  233. pyxlpr/ppocr/tools/program.py +0 -596
  234. pyxlpr/ppocr/tools/test_hubserving.py +0 -117
  235. pyxlpr/ppocr/tools/train.py +0 -163
  236. pyxlpr/ppocr/tools/xlprog.py +0 -748
  237. pyxlpr/ppocr/utils/EN_symbol_dict.txt +0 -94
  238. pyxlpr/ppocr/utils/__init__.py +0 -24
  239. pyxlpr/ppocr/utils/dict/ar_dict.txt +0 -117
  240. pyxlpr/ppocr/utils/dict/arabic_dict.txt +0 -162
  241. pyxlpr/ppocr/utils/dict/be_dict.txt +0 -145
  242. pyxlpr/ppocr/utils/dict/bg_dict.txt +0 -140
  243. pyxlpr/ppocr/utils/dict/chinese_cht_dict.txt +0 -8421
  244. pyxlpr/ppocr/utils/dict/cyrillic_dict.txt +0 -163
  245. pyxlpr/ppocr/utils/dict/devanagari_dict.txt +0 -167
  246. pyxlpr/ppocr/utils/dict/en_dict.txt +0 -63
  247. pyxlpr/ppocr/utils/dict/fa_dict.txt +0 -136
  248. pyxlpr/ppocr/utils/dict/french_dict.txt +0 -136
  249. pyxlpr/ppocr/utils/dict/german_dict.txt +0 -143
  250. pyxlpr/ppocr/utils/dict/hi_dict.txt +0 -162
  251. pyxlpr/ppocr/utils/dict/it_dict.txt +0 -118
  252. pyxlpr/ppocr/utils/dict/japan_dict.txt +0 -4399
  253. pyxlpr/ppocr/utils/dict/ka_dict.txt +0 -153
  254. pyxlpr/ppocr/utils/dict/korean_dict.txt +0 -3688
  255. pyxlpr/ppocr/utils/dict/latin_dict.txt +0 -185
  256. pyxlpr/ppocr/utils/dict/mr_dict.txt +0 -153
  257. pyxlpr/ppocr/utils/dict/ne_dict.txt +0 -153
  258. pyxlpr/ppocr/utils/dict/oc_dict.txt +0 -96
  259. pyxlpr/ppocr/utils/dict/pu_dict.txt +0 -130
  260. pyxlpr/ppocr/utils/dict/rs_dict.txt +0 -91
  261. pyxlpr/ppocr/utils/dict/rsc_dict.txt +0 -134
  262. pyxlpr/ppocr/utils/dict/ru_dict.txt +0 -125
  263. pyxlpr/ppocr/utils/dict/ta_dict.txt +0 -128
  264. pyxlpr/ppocr/utils/dict/table_dict.txt +0 -277
  265. pyxlpr/ppocr/utils/dict/table_structure_dict.txt +0 -2759
  266. pyxlpr/ppocr/utils/dict/te_dict.txt +0 -151
  267. pyxlpr/ppocr/utils/dict/ug_dict.txt +0 -114
  268. pyxlpr/ppocr/utils/dict/uk_dict.txt +0 -142
  269. pyxlpr/ppocr/utils/dict/ur_dict.txt +0 -137
  270. pyxlpr/ppocr/utils/dict/xi_dict.txt +0 -110
  271. pyxlpr/ppocr/utils/dict90.txt +0 -90
  272. pyxlpr/ppocr/utils/e2e_metric/Deteval.py +0 -574
  273. pyxlpr/ppocr/utils/e2e_metric/polygon_fast.py +0 -83
  274. pyxlpr/ppocr/utils/e2e_utils/extract_batchsize.py +0 -87
  275. pyxlpr/ppocr/utils/e2e_utils/extract_textpoint_fast.py +0 -457
  276. pyxlpr/ppocr/utils/e2e_utils/extract_textpoint_slow.py +0 -592
  277. pyxlpr/ppocr/utils/e2e_utils/pgnet_pp_utils.py +0 -162
  278. pyxlpr/ppocr/utils/e2e_utils/visual.py +0 -162
  279. pyxlpr/ppocr/utils/en_dict.txt +0 -95
  280. pyxlpr/ppocr/utils/gen_label.py +0 -81
  281. pyxlpr/ppocr/utils/ic15_dict.txt +0 -36
  282. pyxlpr/ppocr/utils/iou.py +0 -54
  283. pyxlpr/ppocr/utils/logging.py +0 -69
  284. pyxlpr/ppocr/utils/network.py +0 -84
  285. pyxlpr/ppocr/utils/ppocr_keys_v1.txt +0 -6623
  286. pyxlpr/ppocr/utils/profiler.py +0 -110
  287. pyxlpr/ppocr/utils/save_load.py +0 -150
  288. pyxlpr/ppocr/utils/stats.py +0 -72
  289. pyxlpr/ppocr/utils/utility.py +0 -80
  290. pyxlpr/ppstructure/__init__.py +0 -13
  291. pyxlpr/ppstructure/predict_system.py +0 -187
  292. pyxlpr/ppstructure/table/__init__.py +0 -13
  293. pyxlpr/ppstructure/table/eval_table.py +0 -72
  294. pyxlpr/ppstructure/table/matcher.py +0 -192
  295. pyxlpr/ppstructure/table/predict_structure.py +0 -136
  296. pyxlpr/ppstructure/table/predict_table.py +0 -221
  297. pyxlpr/ppstructure/table/table_metric/__init__.py +0 -16
  298. pyxlpr/ppstructure/table/table_metric/parallel.py +0 -51
  299. pyxlpr/ppstructure/table/table_metric/table_metric.py +0 -247
  300. pyxlpr/ppstructure/table/tablepyxl/__init__.py +0 -13
  301. pyxlpr/ppstructure/table/tablepyxl/style.py +0 -283
  302. pyxlpr/ppstructure/table/tablepyxl/tablepyxl.py +0 -118
  303. pyxlpr/ppstructure/utility.py +0 -71
  304. pyxlpr/xlai.py +0 -10
  305. /pyxllib/{ext/autogui → autogui}/virtualkey.py +0 -0
  306. {pyxllib-0.3.96.dist-info → pyxllib-0.3.197.dist-info/licenses}/LICENSE +0 -0
@@ -0,0 +1,341 @@
1
+ #!/usr/bin/env python3
2
+ # -*- coding: utf-8 -*-
3
+ # @Author : 陈坤泽
4
+ # @Email : 877362867@qq.com
5
+ # @Date : 2024/10/12
6
+
7
+ """
8
+ 我们自己的一套利用后端服务进行文件同步的工具
9
+ """
10
+
11
+ import os
12
+ import time
13
+
14
+ from loguru import logger
15
+ from tqdm import tqdm
16
+ import requests
17
+
18
+ from pyxllib.file.specialist import XlPath, GetEtag
19
+ from pyxllib.prog.pupil import format_exception
20
+
21
+
22
+ class SyncFileClient:
23
+ """
24
+
25
+ 注意:该类填写的路径,一律是相对local_root、remote_root的路径,不是当前实际工作目录下的路径!
26
+ 可以通过把local_host设为空的来实现一些特殊精细化的操作
27
+ 但一般推荐还是都把local_root、remote_root设置上
28
+ """
29
+
30
+ def __1_basic(self):
31
+ pass
32
+
33
+ def __init__(self, host, token=None):
34
+ """
35
+ :param host: 可以只写主机名
36
+ """
37
+ token = token or os.getenv("XL_COMMON_SERVER_TOKEN")
38
+ self.headers = {
39
+ 'Authorization': f'Bearer {token}',
40
+ }
41
+ self.local_root = None
42
+ self.remote_root = None
43
+ self.host = self.link_host(host)
44
+
45
+ def link_host(self, hostname):
46
+ from requests.exceptions import Timeout
47
+
48
+ host = f'{os.getenv("MAIN_WEBSITE")}/{hostname}'
49
+ resp = requests.get(f'{host}/common/get_local_server', headers=self.headers)
50
+ ip = resp.json()['ip']
51
+
52
+ if ip:
53
+ try:
54
+ resp = requests.get(f'{ip}/healthz', timeout=5)
55
+ if resp.status_code == 200 and resp.json()['status'] == 'ok':
56
+ host = ip
57
+ except Timeout:
58
+ pass
59
+ except Exception as e:
60
+ raise e
61
+
62
+ remote_root = requests.get(f'{host}/common/get_wkdir', headers=self.headers).json()['wkdir']
63
+ self.remote_root = XlPath(remote_root)
64
+
65
+ return host
66
+
67
+ def get_abs_local_path(self, local_path, remote_path=None):
68
+ # 1 如果需要,从远程路径推测本地路径
69
+ if local_path is None and remote_path is not None:
70
+ local_path = remote_path
71
+
72
+ # 2 本地存储的文件路径
73
+ if self.local_root:
74
+ return self.local_root / local_path
75
+ else:
76
+ return XlPath(local_path)
77
+
78
+ def get_abs_remote_path(self, remote_path, local_path=None):
79
+ # 1 如果需要,从本地路径推测远程路径
80
+ if remote_path is None and local_path is not None:
81
+ remote_path = local_path
82
+
83
+ # 2 远程存储的文件路径
84
+ if self.remote_root:
85
+ return self.remote_root / remote_path
86
+ else:
87
+ return XlPath(remote_path)
88
+
89
+ def __2_sync(self):
90
+ pass
91
+
92
+ def upload_file(self, local_file=None, remote_file=None, relpath=True):
93
+ """
94
+ :param str remote_file: 要存储的远程文件路径,默认可以不指定,自动通过local_file存储到对称位置
95
+
96
+ 使用示例:
97
+ sfc = SyncFileClient('codepc_mi15')
98
+ sfc.upload_file('abc/README.md')
99
+ sfc.upload_file('abc/README.md', 'bcd/1.md')
100
+ """
101
+ file = [local_file]
102
+ if remote_file:
103
+ file.append(remote_file)
104
+ return self.upload_files([file], relpath=relpath)
105
+
106
+ def upload_files(self, files, relpath=True):
107
+ """
108
+ :param list[local_file<, remote_path>] files:
109
+ remote_path可选,未写的时候会自动填充
110
+ :return:
111
+
112
+ todo 加进度条?不过问了下gpt,这个实现好像有点复杂,先不折腾了。本来其实就推荐单个单个文件上传,在外部进行进度展示管控。
113
+
114
+ 使用示例:
115
+ sfc = SyncFileClient('codepc_mi15')
116
+ print(sfc.upload_files(['data/d231120禅宗1期4阶.xlsx', 'data/d231120禅宗1期4阶2.xlsx']))
117
+ """
118
+ # 1 映射出目标路径
119
+ files2 = []
120
+ for file in files:
121
+ if not isinstance(file, (list, tuple)):
122
+ local_file, remote_file = file, None
123
+ else:
124
+ local_file, remote_file = file
125
+
126
+ if relpath:
127
+ local_file = self.get_abs_local_path(local_file, remote_file)
128
+ remote_file = self.get_abs_remote_path(remote_file, local_file)
129
+
130
+ files2.append([local_file, remote_file.as_posix()])
131
+
132
+ files3 = [('files', (remote_file, open(local_file, 'rb'))) for local_file, remote_file in files2]
133
+
134
+ # 4 发送请求
135
+ resp = requests.post(f'{self.host}/common/upload_files',
136
+ headers=self.headers,
137
+ files=files3)
138
+ return resp.json()
139
+
140
+ def download_file(self, remote_file=None, local_file=None, relpath=True):
141
+ """
142
+ :param str remote_file: 要下载的远程文件路径,相对get_wkdir下的路径
143
+ :param str local_file: 下载到本地的文件路径,默认可以不指定,自动通过remote_file存储到对称位置
144
+
145
+ todo 加进度条?方便超大文件下载查看进度。
146
+
147
+ 使用示例:
148
+ sfc = SyncFileClient('http://yourserver.com', headers={"Authorization": "Bearer your_token"})
149
+ sfc.download_file('.vscode/launch.json', 'launch.json')
150
+ """
151
+ if relpath:
152
+ local_file = self.get_abs_local_path(local_file, remote_file)
153
+ remote_file = self.get_abs_remote_path(remote_file, local_file)
154
+
155
+ data = {'file': remote_file.as_posix()}
156
+ # 使用 stream=True 开启流式处理
157
+ with requests.post(f'{self.host}/common/download_file',
158
+ headers=self.headers, json=data, stream=True) as resp:
159
+ if resp.status_code == 200:
160
+ # 以二进制方式写入文件
161
+ with open(local_file, 'wb') as f:
162
+ # 分块写入文件,每次 1MB (1024 * 1024 bytes)
163
+ for chunk in resp.iter_content(chunk_size=1024 * 1024):
164
+ if chunk: # 忽略keep-alive的空块
165
+ f.write(chunk)
166
+ return local_file
167
+ else:
168
+ # 处理其他状态码,抛出异常或进行错误处理
169
+ print(f"Error: Unable to download file, status code {resp.status_code}")
170
+ return None
171
+
172
+ def verify_etag(self, local_file=None, remote_file=None, relpath=True):
173
+ """
174
+ :param relpath: 是否使用的是相对路径
175
+ 当不使用relpath的时候,注意local_file、remote_file都是手动必填的XlPath类型字段
176
+ :return:
177
+ """
178
+ if relpath:
179
+ local_file = self.get_abs_local_path(local_file, remote_file)
180
+ remote_file = self.get_abs_remote_path(remote_file, local_file)
181
+ if not local_file.is_file():
182
+ return {'match': False}
183
+ local_etag = GetEtag.from_file(local_file)
184
+ json_data = {'file': remote_file.as_posix(), 'etag': local_etag}
185
+ resp = requests.post(f'{self.host}/common/check_file_etag',
186
+ headers=self.headers,
187
+ json=json_data)
188
+ return resp.json()
189
+
190
+ def upload_dir(self,
191
+ local_dir=None,
192
+ remote_dir=None,
193
+ *,
194
+ delete_local_file=False,
195
+ verify_etag=False,
196
+ relpath=True):
197
+ """
198
+ :param local_dir: 本地目录
199
+ :param remote_dir: 远程目录
200
+ :param delete_local_file: 每个文件上传成功后,是否删除本地文件
201
+ 默认不删除,一般不删除是配合etag校验使用的
202
+ :param verify_etag: 上传前是否校验etag,如果服务器上对应位置已经有对应etag的文件,其实就不用上传了
203
+
204
+ 该方法已修改以支持上传子目录和空目录
205
+ """
206
+ if relpath:
207
+ # D:\home\chenkunze\data\temp2,举个具体的例子案例值,供参考理解
208
+ local_dir = self.get_abs_local_path(local_dir, remote_dir)
209
+ # /home/chenkunze/data/temp2
210
+ remote_dir = self.get_abs_remote_path(remote_dir, local_dir)
211
+
212
+ for root, dirs, files in os.walk(local_dir):
213
+ # root拿到的是这样的绝对路径:D:\home\chenkunze\data\temp2\数值问题
214
+ # '数值问题'
215
+ subdir = XlPath(root).relpath(local_dir)
216
+ if not files and not dirs:
217
+ # 如果该目录为空,则在远程服务器上创建空目录
218
+ self.create_remote_dir(remote_dir / subdir, relpath=False)
219
+
220
+ for file in tqdm(files, desc=f'上传目录:{root}'):
221
+ local_path = local_dir / subdir / file
222
+ remote_path = remote_dir / subdir / file
223
+ if verify_etag and self.verify_etag(local_path, remote_path, relpath=False):
224
+ pass # 如果etag校验通过,跳过上传
225
+ else:
226
+ self.upload_file(local_path, remote_path, relpath=False)
227
+
228
+ if delete_local_file:
229
+ os.remove(local_path)
230
+
231
+ def create_remote_dir(self, remote_dir, relpath=True):
232
+ """
233
+ 创建远程目录的辅助方法
234
+ """
235
+ if relpath:
236
+ remote_dir = self.get_abs_remote_path(remote_dir)
237
+ json_data = {'dir': remote_dir.as_posix()}
238
+ resp = requests.post(f'{self.host}/common/create_dir',
239
+ headers=self.headers,
240
+ json=json_data)
241
+ return resp.json()
242
+
243
+ def download_dir(self, remote_dir=None, local_dir=None, *, verify_etag=False, relpath=True):
244
+ """
245
+ 下载远程目录到本地
246
+ :param remote_dir: 远程目录路径,相对路径或绝对路径都支持
247
+ :param local_dir: 本地保存目录,默认会自动映射到对应的远程位置
248
+ :param verify_etag: 是否进行etag校验,跳过已经在本地存在并且未修改的文件
249
+ :return:
250
+
251
+ 示例:
252
+ sfc = SyncFileClient('codepc_mi15')
253
+ sfc.download_dir('data/temp2')
254
+ """
255
+ # 1. 获取绝对路径
256
+ if relpath:
257
+ local_dir = self.get_abs_local_path(local_dir, remote_dir)
258
+ remote_dir = self.get_abs_remote_path(remote_dir, local_dir)
259
+
260
+ # 2. 从服务器获取远程目录的文件和子目录结构
261
+ json_data = {'dir': remote_dir.as_posix()}
262
+ resp = requests.post(f'{self.host}/common/list_dir', headers=self.headers, json=json_data)
263
+ if resp.status_code != 200:
264
+ raise Exception(f"Failed to list remote directory: {resp.text}")
265
+
266
+ dir_structure = resp.json() # 期望返回格式为 {'dirs': [...], 'files': [...]}
267
+
268
+ # 3. 创建本地目录
269
+ if not local_dir.exists():
270
+ local_dir.mkdir(parents=True)
271
+
272
+ # 4. 先处理子目录
273
+ for subdir in dir_structure.get('dirs', []):
274
+ local_subdir = local_dir / subdir
275
+ if not local_subdir.exists():
276
+ local_subdir.mkdir(parents=True)
277
+
278
+ # 递归下载子目录中的文件
279
+ for i in range(100): # 开发机senseserver3好像不稳定,加了重试机制
280
+ try:
281
+ self.download_dir(remote_dir / subdir, local_dir / subdir, verify_etag=verify_etag)
282
+ break
283
+ except requests.exceptions.RequestException as e:
284
+ logger.warning(format_exception(e, 3))
285
+ time.sleep(60)
286
+ continue
287
+ else:
288
+ raise requests.exceptions.RequestException
289
+
290
+ # 5. 下载文件
291
+ for file in tqdm(dir_structure.get('files', []), desc=f'下载目录:{remote_dir}'):
292
+ remote_file = remote_dir / file
293
+ local_file = local_dir / file
294
+
295
+ for i in range(100):
296
+ try:
297
+ # 校验etag,如果verify_etag为True并且校验成功,则跳过下载
298
+ if verify_etag and self.verify_etag(local_file, remote_file, relpath=False)['match']:
299
+ break
300
+
301
+ # 下载文件
302
+ logger.info(f'下载文件中:{local_file}')
303
+ self.download_file(remote_file, local_file, relpath=False)
304
+ break
305
+ except requests.exceptions.RequestException:
306
+ time.sleep(60)
307
+ continue
308
+ else:
309
+ raise requests.exceptions.RequestException
310
+
311
+ def download_path(self, remote_path=None, local_path=None, *, verify_etag=False, relpath=True):
312
+ """
313
+ 下载远程文件或目录到本地
314
+ :param remote_path: 远程路径,可以是文件或者目录
315
+ :param local_path: 本地保存路径,默认会自动映射到对应的远程位置
316
+ :param verify_etag: 是否进行etag校验,跳过已经在本地存在并且未修改的文件
317
+ :return:
318
+ """
319
+ if relpath:
320
+ # 获取远程路径的绝对路径
321
+ remote_path = self.get_abs_remote_path(remote_path, local_path)
322
+ local_path = self.get_abs_local_path(local_path, remote_path)
323
+
324
+ # 判断是文件还是目录
325
+ json_data = {'path': remote_path.as_posix()}
326
+ resp = requests.post(f'{self.host}/common/check_path_type', headers=self.headers, json=json_data)
327
+
328
+ if resp.status_code != 200:
329
+ raise Exception(f"Failed to check remote path type: {resp.text}")
330
+
331
+ path_info = resp.json()
332
+
333
+ if path_info['type'] == 'file':
334
+ # 如果是文件,调用 download_file
335
+ self.download_file(remote_file=remote_path, local_file=local_path, relpath=False)
336
+ elif path_info['type'] == 'dir':
337
+ # 如果是目录,调用 download_dir
338
+ self.download_dir(remote_dir=remote_path, local_dir=local_path,
339
+ verify_etag=verify_etag, relpath=False)
340
+ else:
341
+ raise ValueError(f"Unknown path type: {path_info['type']}")
@@ -0,0 +1,64 @@
1
+ #!/usr/bin/env python3
2
+ # -*- coding: utf-8 -*-
3
+ # @Author : 陈坤泽
4
+ # @Email : 877362867@qq.com
5
+ # @Date : 2024/05/29
6
+
7
+ from pyxllib.prog.pupil import check_install_package
8
+
9
+ # cachetools,通用缓存工具,适用范围更广
10
+ check_install_package('cachetools')
11
+ # cached-property,类属性特用工具,相对比较简洁些
12
+ check_install_package('cached_property', 'cached-property')
13
+
14
+ # 对于普通函数,一般用lru_cache即可
15
+ from functools import lru_cache
16
+ import threading
17
+
18
+ from cachetools import cached, LRUCache, TTLCache
19
+
20
+ # 官方文档:https://pypi.org/project/cached-property/
21
+ from cached_property import cached_property
22
+ from cached_property import threaded_cached_property # 线程安全
23
+ from cached_property import cached_property_with_ttl # 限时缓存,单位秒
24
+ from cached_property import threaded_cached_property_with_ttl # 线程 + 限时
25
+
26
+
27
+ # todo 240609周日21:19 https://github.com/awolverp/cachebox,据说这个缓存库速度更快的多
28
+
29
+ # 进一步封装的更通用、自己常用的装饰器
30
+
31
+ def xlcache(maxsize=128, *, ttl=None, lock=None, property=False):
32
+ """ 那些工具接口太复杂难记,自己封装一个统一的工具
33
+
34
+ 就是一个装饰器,最大缓存多少项,然后是否要开多线程安全,是否要设置限时重置,是否是作为类成员属性修饰
35
+
36
+ :param property: 是否作为类成员属性修饰,不过一般不建议通过这里设置,
37
+ 而是外部再加一层@property,不然IDE会识别不了这是一个property,影响开发
38
+
39
+ """
40
+ def decorator(func):
41
+ if property:
42
+ if ttl is not None:
43
+ if lock:
44
+ # 使用带有时间限制和线程安全的缓存属性
45
+ return threaded_cached_property_with_ttl(ttl)(func)
46
+ else:
47
+ # 使用带有时间限制但非线程安全的缓存属性
48
+ return cached_property_with_ttl(ttl)(func)
49
+ else:
50
+ if lock:
51
+ # 使用线程安全的缓存属性
52
+ return threaded_cached_property(func)
53
+ else:
54
+ # 使用普通的缓存属性
55
+ return cached_property(func)
56
+ else:
57
+ lock2 = threading.RLock() if lock is True else lock
58
+ if ttl is None:
59
+ return cached(LRUCache(maxsize), lock=lock2)(func)
60
+ else:
61
+ cache = TTLCache(maxsize=maxsize, ttl=ttl)
62
+ return cached(cache, lock=lock2)(func)
63
+
64
+ return decorator
@@ -0,0 +1,42 @@
1
+ #!/usr/bin/env python3
2
+ # -*- coding: utf-8 -*-
3
+ # @Author : 陈坤泽
4
+ # @Email : 877362867@qq.com
5
+ # @Date : 2024/11/13
6
+
7
+ import os
8
+ import tempfile
9
+ import time
10
+
11
+ from filelock import FileLock, Timeout
12
+
13
+
14
+ class XlFileLock(FileLock):
15
+ def __init__(self, lock_file: str, timeout: float = -1, **kwargs):
16
+ """
17
+ 初始化 XlFileLock 实例,锁文件默认存储在系统临时目录中。
18
+
19
+ :param str filename: 锁文件的文件名(不包含路径)。
20
+ :param float timeout: 获取锁的超时时间(以秒为单位),默认值为 -1 表示无限等待。
21
+ """
22
+ lock_file_path = os.path.join(tempfile.gettempdir(), lock_file)
23
+ super().__init__(lock_file=lock_file_path, timeout=timeout, **kwargs)
24
+
25
+
26
+ def get_autoui_lock(lock_file='autoui.lock', timeout=-1):
27
+ """
28
+ 注意:XlFileLock中的lock_file无法设置默认值,这跟FileLock的底层有关
29
+ 所以无法继承一个AutoUiLock类,但是可以通过这种函数的方式,绕开它特殊的初始化限制
30
+ """
31
+ lock = XlFileLock(lock_file=lock_file, timeout=timeout)
32
+ return lock
33
+
34
+
35
+ if __name__ == "__main__":
36
+ try:
37
+ with get_autoui_lock(timeout=5):
38
+ print("Lock acquired, doing some work...")
39
+ time.sleep(3)
40
+ print("Work done.")
41
+ except Timeout:
42
+ print("Failed to acquire lock within the timeout.")