megfile 3.1.6__py3-none-any.whl → 3.1.7__py3-none-any.whl

megfile/config.py

@@ -60,3 +60,5 @@ HDFS_MAX_RETRY_TIMES = int(
 SFTP_MAX_RETRY_TIMES = int(
     os.getenv("MEGFILE_SFTP_MAX_RETRY_TIMES") or DEFAULT_MAX_RETRY_TIMES
 )
+
+HTTP_AUTH_HEADERS = ("Authorization", "Www-Authenticate", "Cookie", "Cookie2")

megfile/lib/s3_prefetch_reader.py

@@ -76,7 +76,11 @@ class S3PrefetchReader(BasePrefetchReader):
         try:
             start, end = 0, self._block_size - 1
             first_index_response = self._fetch_response(start=start, end=end)
-            content_size = int(first_index_response["ContentRange"].split("/")[-1])
+            if "ContentRange" in first_index_response:
+                content_size = int(first_index_response["ContentRange"].split("/")[-1])
+            else:
+                # usually when read a file only have one block
+                content_size = int(first_index_response["ContentLength"])
         except S3InvalidRangeError:
             # usually when read a empty file
             # can use minio test empty file: https://hub.docker.com/r/minio/minio

megfile/s3_path.py

@@ -7,16 +7,18 @@ from concurrent.futures import ThreadPoolExecutor
 from functools import cached_property, lru_cache, wraps
 from logging import getLogger as get_logger
 from typing import IO, Any, BinaryIO, Callable, Dict, Iterator, List, Optional, Tuple
+from urllib.parse import urlparse
 
 import boto3
 import botocore
-from botocore.awsrequest import AWSResponse
+from botocore.awsrequest import AWSPreparedRequest, AWSResponse
 
 from megfile.config import (
     DEFAULT_BLOCK_SIZE,
     DEFAULT_MAX_BLOCK_SIZE,
     DEFAULT_MIN_BLOCK_SIZE,
     GLOBAL_MAX_WORKERS,
+    HTTP_AUTH_HEADERS,
     S3_CLIENT_CACHE_MODE,
     S3_MAX_RETRY_TIMES,
 )
@@ -76,6 +78,7 @@ from megfile.utils import (
     generate_cache_path,
     get_binary_mode,
     get_content_offset,
+    is_domain_or_subdomain,
     is_readable,
     necessary_params,
     process_local,
@@ -162,24 +165,30 @@ def _patch_make_request(client: botocore.client.BaseClient, redirect: bool = Fal
         retry_callback=retry_callback,
     )
 
-    def patch_send_request(send_request):
-        def patched_send_request(request_dict, operation_model):
-            http, parsed_response = send_request(request_dict, operation_model)
+    def patch_send(send):
+        def patched_send(request: AWSPreparedRequest) -> AWSResponse:
+            response: AWSResponse = send(request)
             if (
-                request_dict["method"] == "GET"  # only support GET method for now
-                and http.status_code in (301, 302, 307, 308)
-                and "Location" in http.headers
+                request.method == "GET"  # only support GET method for now
+                and response.status_code in (301, 302, 307, 308)
+                and "Location" in response.headers
             ):
-                request_dict["url"] = http.headers["Location"]
-                http, parsed_response = send_request(request_dict, operation_model)
-            return http, parsed_response
-
-        return patched_send_request
+                # Permit sending auth/cookie headers from "foo.com" to "sub.foo.com".
+                # See also: https://go.dev/src/net/http/client.go#L980
+                location = response.headers["Location"]
+                ihost = urlparse(request.url).hostname
+                dhost = urlparse(location).hostname
+                if not is_domain_or_subdomain(dhost, ihost):
+                    for name in HTTP_AUTH_HEADERS:
+                        request.headers.pop(name, None)
+                request.url = location
+                response = send(request)
+            return response
+
+        return patched_send
 
     if redirect:
-        client._endpoint._send_request = patch_send_request(
-            client._endpoint._send_request
-        )
+        client._endpoint._send = patch_send(client._endpoint._send)
 
     return client
 

megfile/smart.py

@@ -995,11 +995,12 @@ def smart_load_content(
         return s3_load_content(path, start, stop)
 
     with smart_open(path, "rb") as fd:
-        if start:
+        if start is not None:
             fd.seek(start)
         offset = -1
-        if start and stop:
-            offset = stop - start
+        if stop is not None:
+            offset = stop - (start or 0)  # start may be None
+            assert offset >= 0, "stop should be greater than start"
         return fd.read(offset)  # pytype: disable=bad-return-type
 
 

megfile/utils/__init__.py

@@ -346,3 +346,11 @@ class cached_classproperty(cached_property):
                 val = self.func(cls)
                 setattr(cls, self.attrname, val)
         return val
+
+
+def is_domain_or_subdomain(sub, parent):
+    if sub == parent:
+        return True
+    if sub.endswith(f".{parent}"):
+        return True
+    return False

megfile/version.py

@@ -1 +1 @@
-VERSION = "3.1.6"
+VERSION = "3.1.7"

{megfile-3.1.6.dist-info → megfile-3.1.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: megfile
-Version: 3.1.6
+Version: 3.1.7
 Summary: Megvii file operation library
 Author-email: megvii <megfile@megvii.com>
 Project-URL: Homepage, https://github.com/megvii-research/megfile
@@ -27,10 +27,10 @@ Requires-Dist: requests
 Requires-Dist: paramiko
 Requires-Dist: tqdm
 Requires-Dist: pyyaml
-Provides-Extra: cli
-Requires-Dist: click; extra == "cli"
 Provides-Extra: hdfs
 Requires-Dist: hdfs; extra == "hdfs"
+Provides-Extra: cli
+Requires-Dist: click; extra == "cli"
 
 megfile - Megvii FILE library
 ---

{megfile-3.1.6.dist-info → megfile-3.1.7.dist-info}/RECORD

@@ -1,7 +1,7 @@
 docs/conf.py,sha256=sfDSly5jO8W_RmuAptOIp4hd8dNcO-9a5XrHTbxFnNo,2448
 megfile/__init__.py,sha256=i2Lbq_VxIgppaqwkxG0_H35dRfcjJ4mCYWjprOf4hHo,7318
 megfile/cli.py,sha256=0Sgbz3jeUryVll9Aa6R0MpJdQJUGENvz55YF7Jm1Uxc,23482
-megfile/config.py,sha256=_SkJRaVWUdfW1Q9uX0vao-6YVQKJtfej22Z8DykuRps,2331
+megfile/config.py,sha256=k52eO9YUyYRJ0-bsscAXcfEt8xIAHGOHdLSnEJf6z7k,2411
 megfile/errors.py,sha256=a55qKQgyfiLmV-qnojUFzq2gu9JXpj3ZiC2qVaWyUTA,14160
 megfile/fs.py,sha256=dgj5fW-EEzQNdjMF2tkB5DjXu3iHQbtLi5PSIMxR8fc,11966
 megfile/fs_path.py,sha256=Ffvukc176beH5aQMZXXtwH6ApwLYXPViCIUP0pijgT0,41590
@@ -12,14 +12,14 @@ megfile/http_path.py,sha256=BhMNjQVB85IaCGGIKzgEfY73mAVdCzJP08W1RuGeMRA,16119
 megfile/interfaces.py,sha256=7C53Q2FAVFmOEnplfplvWqHab29HJE5RQnpfdb4loVY,8679
 megfile/pathlike.py,sha256=5VAKIArm2UqrpMBJMoNAEydFxLd1mjCZ8iQnKFUIYu0,31274
 megfile/s3.py,sha256=7SdfLjAePVh-bpRyuj566VB4Qa7KP86rCJGzYANR7wQ,13008
-megfile/s3_path.py,sha256=x-_wxVvVpv56LtDOwRAWJCOq3XMk7oAB6xdi22TBlmY,95029
+megfile/s3_path.py,sha256=kOrP45zQbxCxNQcoovd060QARkP8QWYKd8BQGfxGY2g,95447
 megfile/sftp.py,sha256=vyDnYXX3i1j2fhXMC8YCeX-66MDb9wrBQQjQVhZx0uo,13004
 megfile/sftp_path.py,sha256=4tByWvUJK1KBJoa3t5aoWYnZpaRWN9nQIE6ZyiGHrbk,53519
-megfile/smart.py,sha256=Vr4R7HpjXjt587KOc2-1QGbQ5EsZ48YRzCaK0rz3IS0,36108
+megfile/smart.py,sha256=vfWhPgL26KrF4s9K5oV55R3zvAyCr9nvD9CsYFWVA9g,36218
 megfile/smart_path.py,sha256=Wsn6fR9g7NTwNwwvZ_0H39NLHIlOLnCqK-ZY0n5CvKk,7812
 megfile/stdio.py,sha256=UYe-h440Wc4f5COOzOTG1svnp5nFzrfpixehJ0_0_NY,653
 megfile/stdio_path.py,sha256=7jzVdreamO18yBWZM7Pp71cO7GmrYb0M0qyQde2Ypq4,2706
-megfile/version.py,sha256=GfRGwuY_9YYywKtd8Nc8lbkE6hlfX3NUsNtuHN-c8Gs,19
+megfile/version.py,sha256=n9Z_RgYWmzkUHtPVhIWP6RlF4L6HnKS2N0unpIEw86U,19
 megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 megfile/lib/base_prefetch_reader.py,sha256=CaYWuiKmlk4Utr0IFDPwPC58wV2jBAhqpxhwhRHc734,13652
 megfile/lib/combine_reader.py,sha256=uSzo3PmhD5ck6_Vv6dFU5vVx4boeA97VS-puPyhF_BE,4657
@@ -37,19 +37,19 @@ megfile/lib/s3_cached_handler.py,sha256=QrQKck06ye16o7GD71T-fVCseKlOhsxp82LtBTtA
 megfile/lib/s3_limited_seekable_writer.py,sha256=v-e7rfFBfWCSQVtJIaFHM_i0Hb1FkfVLHlhawo5MOIk,6358
 megfile/lib/s3_memory_handler.py,sha256=NGKWbI4LG2cmV06CP7KOVPqS_BNpm3ApqKi5ibgIBvQ,4208
 megfile/lib/s3_pipe_handler.py,sha256=DY1UTNCq8oD3QWXNb4orOiz3EoEAo6dhwmZZdk6h1bU,3694
-megfile/lib/s3_prefetch_reader.py,sha256=YZA6JOQXcioREh_z1E-kZ2WRPTm02v0dCEVqyaOMHns,4287
+megfile/lib/s3_prefetch_reader.py,sha256=gjnnYI95LFwxpneDFfLBzw8gvT1Vc8yJJlBL101oysI,4501
 megfile/lib/s3_share_cache_reader.py,sha256=jhGL1B6NPv68cQnW1Jf7ey-zTQ8XfiJg5ILDNgRWHy0,3671
 megfile/lib/shadow_handler.py,sha256=TntewlvIW9ZxCfmqASDQREHoiZ8v42faOe9sovQYQz0,2779
 megfile/lib/stdio_handler.py,sha256=IDdgENLQlhigEwkLL4zStueVSzdWg7xVcTF_koof_Ek,1987
 megfile/lib/url.py,sha256=ER32pWy9Q2MAk3TraAaNEBWIqUeBmLuM57ol2cs7-Ks,103
-megfile/utils/__init__.py,sha256=9rD_SoD--XWt7-EJi5-L80Y7YeoFdr-tUp-5ATB85oA,10717
+megfile/utils/__init__.py,sha256=RAj8dAJZX5TkWKJu3Ip78uhA5XZ8wpir61eCm6bAnd4,10874
 megfile/utils/mutex.py,sha256=asb8opGLgK22RiuBJUnfsvB8LnMmodP8KzCVHKmQBWA,2561
 scripts/convert_results_to_sarif.py,sha256=nDiOfsedb22Ps7ZodmYdlXZlxv54fRxCQgOZsB2OkNk,2833
 scripts/generate_file.py,sha256=-mTcBiqiQ1juvqojVfVZ-uZWgpANHJNdhrF7s68zNfc,10903
-megfile-3.1.6.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-megfile-3.1.6.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
-megfile-3.1.6.dist-info/METADATA,sha256=XuomkBWbASrj60L6untH8oN32sNezHse1WfuPetTpYg,9178
-megfile-3.1.6.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
-megfile-3.1.6.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
-megfile-3.1.6.dist-info/top_level.txt,sha256=oTnYXo1Z3V61qSWAKtnY9RkDgRSHvfRN38FQae6E0W0,50
-megfile-3.1.6.dist-info/RECORD,,
+megfile-3.1.7.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+megfile-3.1.7.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
+megfile-3.1.7.dist-info/METADATA,sha256=IOh1sm_PB6oLFwTJwWMZx3tvIfyiIFaYG6d0IPTlt1Q,9178
+megfile-3.1.7.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+megfile-3.1.7.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
+megfile-3.1.7.dist-info/top_level.txt,sha256=fVg49lk5B9L7jyfWUXWxb0DDSuw5pbr0OU62Tvx8J8M,44
+megfile-3.1.7.dist-info/RECORD,,

{megfile-3.1.6.dist-info → megfile-3.1.7.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.5.0)
+Generator: setuptools (75.6.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{megfile-3.1.6.dist-info → megfile-3.1.7.dist-info}/top_level.txt

@@ -1,6 +1,5 @@
 dist
 docs
-empty
 html_cov
 html_doc
 megfile