megfile 4.2.4__py3-none-any.whl → 5.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- megfile/__init__.py +16 -291
- megfile/cli.py +37 -20
- megfile/config.py +10 -1
- megfile/errors.py +2 -2
- megfile/fs_path.py +78 -12
- megfile/interfaces.py +44 -0
- megfile/lib/base_memory_handler.py +92 -0
- megfile/lib/glob.py +3 -3
- megfile/lib/http_prefetch_reader.py +22 -22
- megfile/lib/joinpath.py +13 -0
- megfile/lib/s3_buffered_writer.py +13 -0
- megfile/lib/s3_limited_seekable_writer.py +2 -0
- megfile/lib/s3_memory_handler.py +14 -81
- megfile/lib/webdav_memory_handler.py +83 -0
- megfile/lib/webdav_prefetch_reader.py +115 -0
- megfile/pathlike.py +3 -4
- megfile/s3_path.py +44 -33
- megfile/sftp2_path.py +44 -62
- megfile/sftp_path.py +239 -2
- megfile/smart.py +70 -29
- megfile/smart_path.py +181 -85
- megfile/version.py +1 -1
- megfile/webdav_path.py +952 -0
- {megfile-4.2.4.dist-info → megfile-5.0.0.dist-info}/METADATA +30 -39
- megfile-5.0.0.dist-info/RECORD +51 -0
- megfile/fs.py +0 -614
- megfile/hdfs.py +0 -408
- megfile/http.py +0 -114
- megfile/s3.py +0 -540
- megfile/sftp.py +0 -821
- megfile/sftp2.py +0 -827
- megfile/stdio.py +0 -30
- megfile-4.2.4.dist-info/RECORD +0 -54
- {megfile-4.2.4.dist-info → megfile-5.0.0.dist-info}/WHEEL +0 -0
- {megfile-4.2.4.dist-info → megfile-5.0.0.dist-info}/entry_points.txt +0 -0
- {megfile-4.2.4.dist-info → megfile-5.0.0.dist-info}/licenses/LICENSE +0 -0
- {megfile-4.2.4.dist-info → megfile-5.0.0.dist-info}/licenses/LICENSE.pyre +0 -0
- {megfile-4.2.4.dist-info → megfile-5.0.0.dist-info}/top_level.txt +0 -0
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: megfile
|
|
3
|
-
Version:
|
|
3
|
+
Version: 5.0.0
|
|
4
4
|
Summary: Megvii file operation library
|
|
5
|
-
Author-email: megvii <
|
|
5
|
+
Author-email: megvii-reng <megvii-reng@googlegroups.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/megvii-research/megfile
|
|
7
7
|
Classifier: Development Status :: 5 - Production/Stable
|
|
8
8
|
Classifier: Environment :: Console
|
|
@@ -12,12 +12,12 @@ Classifier: Operating System :: POSIX :: Linux
|
|
|
12
12
|
Classifier: Programming Language :: Python
|
|
13
13
|
Classifier: Programming Language :: Python :: 3
|
|
14
14
|
Classifier: Programming Language :: Python :: 3 :: Only
|
|
15
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
16
15
|
Classifier: Programming Language :: Python :: 3.10
|
|
17
16
|
Classifier: Programming Language :: Python :: 3.11
|
|
18
17
|
Classifier: Programming Language :: Python :: 3.12
|
|
19
18
|
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
-
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
20
|
+
Requires-Python: >=3.10
|
|
21
21
|
Description-Content-Type: text/markdown
|
|
22
22
|
License-File: LICENSE
|
|
23
23
|
License-File: LICENSE.pyre
|
|
@@ -31,6 +31,9 @@ Provides-Extra: hdfs
|
|
|
31
31
|
Requires-Dist: hdfs; extra == "hdfs"
|
|
32
32
|
Provides-Extra: sftp2
|
|
33
33
|
Requires-Dist: ssh2-python>=1.0.0; extra == "sftp2"
|
|
34
|
+
Provides-Extra: webdav
|
|
35
|
+
Requires-Dist: webdavclient3>=3.0.0; extra == "webdav"
|
|
36
|
+
Requires-Dist: python-dateutil; extra == "webdav"
|
|
34
37
|
Provides-Extra: cli
|
|
35
38
|
Requires-Dist: click; extra == "cli"
|
|
36
39
|
Dynamic: license-file
|
|
@@ -48,21 +51,16 @@ megfile - Megvii FILE library
|
|
|
48
51
|
|
|
49
52
|
* Docs: http://megvii-research.github.io/megfile
|
|
50
53
|
|
|
51
|
-
`megfile` provides a silky operation experience with different backends
|
|
54
|
+
`megfile` provides a silky operation experience with different backends, which enable you to focus more on the logic of your own project instead of the question of "Which backend is used for this file?"
|
|
52
55
|
|
|
53
|
-
|
|
56
|
+
## Why megfile
|
|
54
57
|
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
*
|
|
58
|
-
*
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
* `smart_open` can open resources that use various protocols. Especially, reader / writer of s3 in `megfile` is implemented with multi-thread, which is faster than known competitors.
|
|
63
|
-
* `smart_glob` is available on majority protocols. And it supports zsh extended pattern syntax of `[]`, e.g. `s3://bucket/video.{mp4,avi}`.
|
|
64
|
-
* All-inclusive functions like `smart_exists` / `smart_stat` / `smart_sync`. If you don't find the functions you want, [submit an issue](https://github.com/megvii-research/megfile/issues).
|
|
65
|
-
* Compatible with `pathlib.Path` interface, referring to `SmartPath` and other protocol classes like `S3Path`.
|
|
58
|
+
* Same interfaces as the python standard library, low learning curve
|
|
59
|
+
* Faster file read and write operations
|
|
60
|
+
* Excellent error retry mechanism to help you handle network issues
|
|
61
|
+
* Supports popular protocols, even making it easy to use the same protocol with different endpoints
|
|
62
|
+
* Stable and secure, with CI coverage over 95%, used by multiple industry giants
|
|
63
|
+
* Perfect type hints and built-in documentation. You can enjoy the IDE's auto-completion and static checking
|
|
66
64
|
|
|
67
65
|
## Support Protocols
|
|
68
66
|
- fs(local filesystem)
|
|
@@ -70,11 +68,15 @@ megfile - Megvii FILE library
|
|
|
70
68
|
- sftp
|
|
71
69
|
- http
|
|
72
70
|
- stdio
|
|
73
|
-
- hdfs: `
|
|
71
|
+
- hdfs: `pip3 install 'megfile[hdfs]'`
|
|
72
|
+
- webdav: `pip3 install 'megfile[webdav]'`
|
|
74
73
|
|
|
75
74
|
## Quick Start
|
|
76
75
|
|
|
77
|
-
|
|
76
|
+
The interfaces of `megfile` correspond to those in the Python standard library. For example, `open` -> `smart_open` and `pathlib.Path` -> `SmartPath`. You only need to [configure the protocol settings](https://megvii-research.github.io/megfile/configuration.html) and provide the path in the corresponding format to use them conveniently.
|
|
77
|
+
|
|
78
|
+
Path string in `megfile` almost is `protocol://path/to/file`, for example `s3://bucketA/key`. More details see [path format document](https://megvii-research.github.io/megfile/path_format.html).
|
|
79
|
+
|
|
78
80
|
Here's an example of writing a file to s3 / fs, syncing to local, reading and finally deleting it.
|
|
79
81
|
|
|
80
82
|
### Functional Interface
|
|
@@ -103,7 +105,7 @@ smart_glob('s3://playground/megfile-?.{mp4,avi}')
|
|
|
103
105
|
|
|
104
106
|
### SmartPath Interface
|
|
105
107
|
|
|
106
|
-
`SmartPath` has a similar interface with pathlib.Path
|
|
108
|
+
`SmartPath` has a similar interface with `pathlib.Path`.
|
|
107
109
|
|
|
108
110
|
```python
|
|
109
111
|
from megfile.smart_path import SmartPath
|
|
@@ -135,28 +137,15 @@ $ megfile cp s3://playground/megfile-test /tmp/playground/megfile-test
|
|
|
135
137
|
|
|
136
138
|
```bash
|
|
137
139
|
pip3 install megfile
|
|
138
|
-
```
|
|
139
140
|
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
pip3 install "megfile~=0.0"
|
|
143
|
-
```
|
|
141
|
+
# for cli support
|
|
142
|
+
pip3 install 'megfile[cli]'
|
|
144
143
|
|
|
145
|
-
|
|
144
|
+
# for hdfs support
|
|
145
|
+
pip3 install 'megfile[hdfs]'
|
|
146
146
|
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
git clone git@github.com:megvii-research/megfile.git
|
|
150
|
-
cd megfile
|
|
151
|
-
pip3 install -U .
|
|
152
|
-
```
|
|
153
|
-
|
|
154
|
-
### Development Environment
|
|
155
|
-
|
|
156
|
-
```bash
|
|
157
|
-
git clone git@github.com:megvii-research/megfile.git
|
|
158
|
-
cd megfile
|
|
159
|
-
pip3 install -r requirements.txt -r requirements-dev.txt
|
|
147
|
+
# for webdav support
|
|
148
|
+
pip3 install 'megfile[webdav]'
|
|
160
149
|
```
|
|
161
150
|
|
|
162
151
|
## Configuration
|
|
@@ -215,6 +204,8 @@ You can get the configuration from `~/.config/megfile/aliases.conf`, like:
|
|
|
215
204
|
protocol = s3+tos
|
|
216
205
|
```
|
|
217
206
|
|
|
207
|
+
You can use alias in path, like `tos://bucket/key`, the same as `s3+tos://bucket/key`.
|
|
208
|
+
|
|
218
209
|
## Benchmark
|
|
219
210
|
[](https://megvii-research.github.io/megfile/benchmark.html)
|
|
220
211
|
[](https://megvii-research.github.io/megfile/benchmark.html)
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
megfile/__init__.py,sha256=2QtWAC9XFlKDhY0NTHsCJd2wQ-8LKz0q1ng4-8cXo38,2836
|
|
2
|
+
megfile/cli.py,sha256=yRrnXLYejCz1928JCjjiqOe50YH8lD1xuwaiej_67Ng,29352
|
|
3
|
+
megfile/config.py,sha256=r_QmInU_gFiSTV1nLJ5SyBVCX5bsANqOoNM82bRESnQ,4414
|
|
4
|
+
megfile/errors.py,sha256=eC7z-2-QqE12pYgTGAOIlEBsemqcMyeBFVUEDz3gBS0,15585
|
|
5
|
+
megfile/fs_path.py,sha256=tt2__W6E4vep0lmVreTLIW63njl-EzyQEEkEGziyAb4,41015
|
|
6
|
+
megfile/hdfs_path.py,sha256=OmUe3vA3Qoxnqtcq0Rs3ygBvzAtqUz3fGo8iP5sWneE,26058
|
|
7
|
+
megfile/http_path.py,sha256=08OmzmRMyLSyq1Yr1K2HbzexesURJrIoA6AibwYzUiA,13844
|
|
8
|
+
megfile/interfaces.py,sha256=XU46U5pl4k1Gse63i4z5SvxcjWeKLj0xyB0Y6fYiWWo,9887
|
|
9
|
+
megfile/pathlike.py,sha256=kacTG55j3dq3b7YP7MT7jxeJGbDwvK3XuZIvF-UUE0o,31244
|
|
10
|
+
megfile/s3_path.py,sha256=LINHnHnpesXnf9wxbV6n0xQVT0wPwyjLc7xAasakefU,94467
|
|
11
|
+
megfile/sftp2_path.py,sha256=K90bnMVAx0MQPGXP6LogGuDRzaD4MPR6lMOfdY9C9-0,37942
|
|
12
|
+
megfile/sftp_path.py,sha256=_KU7_-Mq2m7lcLY1mpiGrju0SP-OsdEXlRjFhZH25UA,51223
|
|
13
|
+
megfile/smart.py,sha256=Lab2jxprj-zvPw5GqUWlWiEY8bcpRlviks_qp9r-km8,38224
|
|
14
|
+
megfile/smart_path.py,sha256=cI8LSD1M7p-pHFJ-yDIgen8g9FyfbfDLrDfQwkkKT-Q,12240
|
|
15
|
+
megfile/stdio_path.py,sha256=cxaDr8rtisTPnN-rjtaEpqQnshwiqwXFUJBM9xWY7Cg,2711
|
|
16
|
+
megfile/version.py,sha256=sWZ_cOSUdsW9iQos7P4NME_qRop6_sbMdttJcWoYa7o,19
|
|
17
|
+
megfile/webdav_path.py,sha256=96rt3jN_XfDXY0UYC_Kufrz5Uc5KySo34QnlVbDIQK8,31278
|
|
18
|
+
megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
19
|
+
megfile/lib/base_memory_handler.py,sha256=i5-YHpL0k2tuFBnP9VMzb3_OsaU6D6j6thkmWgptnFg,2680
|
|
20
|
+
megfile/lib/base_prefetch_reader.py,sha256=MYaWOkXc3geZMYNPHlPZmmpOR6uSz-AMuCZwYdoz7t0,13296
|
|
21
|
+
megfile/lib/combine_reader.py,sha256=Kp2wEloOUpTlIU7dve87MBpSzmIM-F9OtpTawAjFkiU,4828
|
|
22
|
+
megfile/lib/compare.py,sha256=CPSbyqsQ396oSfxa7h0NdUUqBw5A3WOn6fHrNKkuinw,2188
|
|
23
|
+
megfile/lib/compat.py,sha256=SynEeHluys3tCK-lb_1oV3o_ft83yZvunqM_AjibLgE,207
|
|
24
|
+
megfile/lib/fnmatch.py,sha256=4MvGzEahMRA-u8Z7mxaD-Yw1idOwBoJJpVywQy29jwY,4162
|
|
25
|
+
megfile/lib/glob.py,sha256=Xt2lSEMn9FmffmH68Y2rlpJhq7K3pN2aOtZpw3byW74,9793
|
|
26
|
+
megfile/lib/hdfs_prefetch_reader.py,sha256=yCNpcXcTiC2SHKHC-Qp50KQx1ObSLmOgwNUKlG-4ADg,2131
|
|
27
|
+
megfile/lib/hdfs_tools.py,sha256=4K-OdMYFFSLBGmDzjatioHvuZuUbKVy7ACeJl-l0HLQ,435
|
|
28
|
+
megfile/lib/http_prefetch_reader.py,sha256=x0m5tKN8QLl5QkZyNEDW662UJaZoMf61GDHVvXsDrTQ,4030
|
|
29
|
+
megfile/lib/joinpath.py,sha256=R3sz3pvcgbv1e793vZUnwGH-NmDlEghEPNwq5IEMm4I,1251
|
|
30
|
+
megfile/lib/lazy_handler.py,sha256=bE7RGt1x_xYWMgGAvHr7dwEt52qy-D3z90X3oyCvE6g,1875
|
|
31
|
+
megfile/lib/s3_buffered_writer.py,sha256=a42tr48QXToTQtZD1XfK1Veju6qLCLF0RJC0V_P3pW8,8252
|
|
32
|
+
megfile/lib/s3_cached_handler.py,sha256=MkNt6AAapd5x8BH2gnW5_S0cLofN-mshEdb0qSoLho8,1426
|
|
33
|
+
megfile/lib/s3_limited_seekable_writer.py,sha256=rBlGCsrIJdVRKdsJ1uIAE_R6EN96Kl2JMskk-5czYmE,6289
|
|
34
|
+
megfile/lib/s3_memory_handler.py,sha256=YHakyN21hvBoSlGXSV1UfFMFBh3WJNYWmiwyEzmIytw,2125
|
|
35
|
+
megfile/lib/s3_pipe_handler.py,sha256=g3iAN1P9pCdvSNsGeJBGcBa10S62oqIg_9W3b3wc7os,3809
|
|
36
|
+
megfile/lib/s3_prefetch_reader.py,sha256=AqfADmbbZYA6nw4vxBOiFWX5q5CSYOd0hq1LWcf1PY0,4524
|
|
37
|
+
megfile/lib/s3_share_cache_reader.py,sha256=8uip5IdVjPXCquXrskjocsZx2-TiXqWZPY0gX8JC144,4020
|
|
38
|
+
megfile/lib/shadow_handler.py,sha256=TntewlvIW9ZxCfmqASDQREHoiZ8v42faOe9sovQYQz0,2779
|
|
39
|
+
megfile/lib/stdio_handler.py,sha256=IDdgENLQlhigEwkLL4zStueVSzdWg7xVcTF_koof_Ek,1987
|
|
40
|
+
megfile/lib/url.py,sha256=ER32pWy9Q2MAk3TraAaNEBWIqUeBmLuM57ol2cs7-Ks,103
|
|
41
|
+
megfile/lib/webdav_memory_handler.py,sha256=KDSU-Z3ECiH5p6mo8TsbqHxjSBCZNx-LtFW9nIimdOY,2508
|
|
42
|
+
megfile/lib/webdav_prefetch_reader.py,sha256=M0X6E6t-DS5q9KiLvjVZx_AZuiW9SaIkBnIPLc774GQ,3941
|
|
43
|
+
megfile/utils/__init__.py,sha256=4hBVSXbNTbDj7Je0y9SbwgcPm_s41H9v3eHUMr9JNGo,12700
|
|
44
|
+
megfile/utils/mutex.py,sha256=asb8opGLgK22RiuBJUnfsvB8LnMmodP8KzCVHKmQBWA,2561
|
|
45
|
+
megfile-5.0.0.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
|
|
46
|
+
megfile-5.0.0.dist-info/licenses/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
|
|
47
|
+
megfile-5.0.0.dist-info/METADATA,sha256=xwRH2pja42oQN9jJHaiTsFeKO0Ib8Ih4Thvi2cyNns0,9225
|
|
48
|
+
megfile-5.0.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
49
|
+
megfile-5.0.0.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
|
|
50
|
+
megfile-5.0.0.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
|
|
51
|
+
megfile-5.0.0.dist-info/RECORD,,
|