unzip-http 0.4__tar.gz → 0.6__tar.gz

unzip-http-0.6/PKG-INFO

@@ -0,0 +1,50 @@
+Metadata-Version: 2.1
+Name: unzip-http
+Version: 0.6
+Summary: extract files from .zip files over http without downloading entire archive
+Home-page: https://github.com/saulpw/unzip-http
+Author: Saul Pwanson
+Keywords: http zip unzip
+Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE-mit.txt
+
+# unzip-http
+
+Extract individual files from .zip files over http without downloading the entire archive.
+
+## Install
+
+    pip install unzip-http
+
+## Usage
+
+    unzip_http [-l] [-f] [-o] <url> <filenames..>
+
+Extract <filenames> from a remote .zip at `<url>` to stdout.
+
+A filename can be a wildcard glob; all matching files are extracted in this case.
+
+Specify multiple filenames as distinct arguments (separated with spaces on the command line).
+
+Note: HTTP server must send `Accept-Ranges: bytes` and `Content-Length` in headers (most do).
+
+Options:
+
+- `-l`: List files in remote .zip file (default if no filenames given)
+- `-f`: Recreate folder structure from .zip file when extracting (instead of extracting files to the current directly to the current directory)
+- `-o`: Write files to stdout (if multiple files, concatenate them in zipfile order)
+
+# Python module `unzip_http`
+
+    import unzip_http
+
+    rzf = unzip_http.RemoteZipFile('https://example.com/foo.zip')
+    binfp = rzf.open('bar.bin')
+    txtfp = rzf.open_text('baz.txt')
+
+# Credits
+
+`unzip-http` was written by [Saul Pwanson](https://saul.pw) and made available for use under the MIT License.

{unzip-http-0.4 → unzip-http-0.6}/README.md

@@ -8,15 +8,21 @@ Extract individual files from .zip files over http without downloading the entir
 
 ## Usage
 
-    unzip-http <url.zip> <filenames..>
+    unzip_http [-l] [-f] [-o] <url> <filenames..>
 
 Extract <filenames> from a remote .zip at `<url>` to stdout.
 
-If no filenames given, displays .zip contents (filenames and sizes).
+A filename can be a wildcard glob; all matching files are extracted in this case.
 
-Each filename can be a wildcard glob; all matching files are concatenated and sent to stdout in zipfile order.
+Specify multiple filenames as distinct arguments (separated with spaces on the command line).
 
-Note: HTTP server must send `Accept-Ranges: bytes` and `Content-Length` in headers.
+Note: HTTP server must send `Accept-Ranges: bytes` and `Content-Length` in headers (most do).
+
+Options:
+
+- `-l`: List files in remote .zip file (default if no filenames given)
+- `-f`: Recreate folder structure from .zip file when extracting (instead of extracting files to the current directly to the current directory)
+- `-o`: Write files to stdout (if multiple files, concatenate them in zipfile order)
 
 # Python module `unzip_http`
 

{unzip-http-0.4 → unzip-http-0.6}/setup.py

@@ -14,7 +14,7 @@ def requirements():
 
 setup(
         name="unzip-http",
-        version="0.4",
+        version="0.6",
         description="extract files from .zip files over http without downloading entire archive",
         long_description=readme(),
         long_description_content_type="text/markdown",
@@ -27,6 +27,9 @@ setup(
         url="https://github.com/saulpw/unzip-http",
         python_requires=">=3.8",
         py_modules=["unzip_http"],
-        scripts=["unzip-http"],
+        scripts=["unzip_http.py"],
+        entry_points={
+            "console_scripts": ["unzip_http=unzip_http:main"],
+            },
         install_requires=requirements(),
 )

unzip-http-0.6/unzip_http.egg-info/PKG-INFO

@@ -0,0 +1,50 @@
+Metadata-Version: 2.1
+Name: unzip-http
+Version: 0.6
+Summary: extract files from .zip files over http without downloading entire archive
+Home-page: https://github.com/saulpw/unzip-http
+Author: Saul Pwanson
+Keywords: http zip unzip
+Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE-mit.txt
+
+# unzip-http
+
+Extract individual files from .zip files over http without downloading the entire archive.
+
+## Install
+
+    pip install unzip-http
+
+## Usage
+
+    unzip_http [-l] [-f] [-o] <url> <filenames..>
+
+Extract <filenames> from a remote .zip at `<url>` to stdout.
+
+A filename can be a wildcard glob; all matching files are extracted in this case.
+
+Specify multiple filenames as distinct arguments (separated with spaces on the command line).
+
+Note: HTTP server must send `Accept-Ranges: bytes` and `Content-Length` in headers (most do).
+
+Options:
+
+- `-l`: List files in remote .zip file (default if no filenames given)
+- `-f`: Recreate folder structure from .zip file when extracting (instead of extracting files to the current directly to the current directory)
+- `-o`: Write files to stdout (if multiple files, concatenate them in zipfile order)
+
+# Python module `unzip_http`
+
+    import unzip_http
+
+    rzf = unzip_http.RemoteZipFile('https://example.com/foo.zip')
+    binfp = rzf.open('bar.bin')
+    txtfp = rzf.open_text('baz.txt')
+
+# Credits
+
+`unzip-http` was written by [Saul Pwanson](https://saul.pw) and made available for use under the MIT License.

{unzip-http-0.4 → unzip-http-0.6}/unzip_http.egg-info/SOURCES.txt

@@ -3,10 +3,10 @@ MANIFEST.in
 README.md
 requirements.txt
 setup.py
-unzip-http
 unzip_http.py
 unzip_http.egg-info/PKG-INFO
 unzip_http.egg-info/SOURCES.txt
 unzip_http.egg-info/dependency_links.txt
+unzip_http.egg-info/entry_points.txt
 unzip_http.egg-info/requires.txt
 unzip_http.egg-info/top_level.txt

unzip-http-0.6/unzip_http.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+unzip_http = unzip_http:main

{unzip-http-0.4 → unzip-http-0.6}/unzip_http.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 # Copyright (c) 2022 Saul Pwanson
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -18,22 +20,50 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
+"""
+usage: unzip_http [-h] [-l] [-f] [-o] url [files ...]
+
+Extract individual files from .zip files over http without downloading the
+entire archive. HTTP server must send `Accept-Ranges: bytes` and
+`Content-Length` in headers.
+
+positional arguments:
+  url                   URL of the remote zip file
+  files                 Files to extract. If no filenames given, displays .zip
+                        contents (filenames and sizes). Each filename can be a
+                        wildcard glob.
+
+options:
+  -h, --help            show this help message and exit
+  -l, --list            List files in the remote zip file
+  -f, --full-filepaths  Recreate folder structure from zip file when extracting
+                        (instead of extracting the files to the current
+                        directory)
+  -o, --stdout          Write files to stdout (if multiple files: concatenate
+                        them to stdout, in zipfile order)
+"""
+
+import sys
 import os
 import io
+import math
+import time
 import zlib
 import struct
 import fnmatch
+import argparse
 import pathlib
 import urllib.parse
 
-import urllib3
 
-__version__ = '0.4'
+__version__ = '0.6'
 
 
 def error(s):
     raise Exception(s)
 
+def warning(s):
+    print(s, file=sys.stderr)
 
 def get_bits(val:int, *args):
     'Generate bitfields (one for each arg) from LSB to MSB.'
@@ -59,6 +89,9 @@ class RemoteZipInfo:
         sec, mins, hour, day, mon, year = get_bits(date_time, 5, 6, 5, 5, 4, 7)
         self.date_time = (year+1980, mon, day, hour, mins, sec)
 
+    def is_dir(self):
+        return self.filename.endswith('/')
+
     def parse_extra(self, extra):
         i = 0
         while i < len(extra):
@@ -93,6 +126,7 @@ class RemoteZipFile:
     magic_eocd = b'\x50\x4b\x05\x06'
 
     def __init__(self, url):
+        import urllib3
         self.url = url
         self.http = urllib3.PoolManager()
         self.zip_size = 0
@@ -120,10 +154,13 @@ class RemoteZipFile:
         r = resp.headers.get('Accept-Ranges', '')
         if r != 'bytes':
             hostname = urllib.parse.urlparse(self.url).netloc
-            error(f"{hostname} does not support HTTP range requests (Accept-Ranges header ('{r}') is not 'bytes')")
+            warning(f"{hostname} Accept-Ranges header ('{r}') is not 'bytes'--trying anyway")
 
         self.zip_size = int(resp.headers['Content-Length'])
-        resp = self.get_range(self.zip_size-65536, 65536)
+        resp = self.get_range(
+            max(self.zip_size-65536, 0),
+            65536
+        )
 
         cdir_start = -1
         i = resp.data.rfind(self.magic_eocd64)
@@ -140,7 +177,15 @@ class RemoteZipFile:
         if cdir_start < 0 or cdir_start >= self.zip_size:
             error('cannot find central directory')
 
-        filehdr_index = 65536 - (self.zip_size - cdir_start)
+        if self.zip_size <= 65536:
+            filehdr_index = cdir_start
+        else:
+            filehdr_index = 65536 - (self.zip_size - cdir_start)
+
+        if filehdr_index < 0:
+            resp = self.get_range(cdir_start, self.zip_size - cdir_start)
+            filehdr_index = 0
+
         cdir_end = filehdr_index + cdir_bytes
         while filehdr_index < cdir_end:
             sizeof_cdirentry = struct.calcsize(self.fmt_cdirentry)
@@ -166,19 +211,26 @@ class RemoteZipFile:
             rzi.parse_extra(extra)
             yield rzi
 
-    def extractall(self, path, members=None):
-        path = path or pathlib.Path('.')
-        for fn in members or self.namelist():
-            outpath = path/fn
+    def extract(self, member, path=None, pwd=None):
+            if pwd:
+                raise NotImplementedError('Passwords not supported yet')
+
+            path = path or pathlib.Path('.')
+
+            outpath = path/member
             os.makedirs(outpath.parent, exist_ok=True)
-            with self.open(fn) as fpin:
-                with open(path/fn, mode='wb') as fpout:
+            with self.open(member) as fpin:
+                with open(path/member, mode='wb') as fpout:
                     while True:
                         r = fpin.read(65536)
                         if not r:
                             break
                         fpout.write(r)
 
+    def extractall(self, path=None, members=None, pwd=None):
+        for fn in members or self.namelist():
+            self.extract(fn, path, pwd=pwd)
+
     def get_range(self, start, n):
         return self.http.request('GET', self.url, headers={'Range': f'bytes={start}-{start+n-1}'}, preload_content=False)
 
@@ -204,12 +256,12 @@ class RemoteZipFile:
             return self.get_range(f.header_offset + sizeof_localhdr + fnlen + extralen, f.compress_size)
         elif method == 8: # DEFLATE
             resp = self.get_range(f.header_offset + sizeof_localhdr + fnlen + extralen, f.compress_size)
-            return RemoteZipStream(resp, f)
+            return io.BufferedReader(RemoteZipStream(resp, f))
         else:
             error(f'unknown compression method {method}')
 
     def open_text(self, fn):
-        return io.TextIOWrapper(io.BufferedReader(self.open(fn)))
+        return io.TextIOWrapper(self.open(fn))
 
 
 class RemoteZipStream(io.RawIOBase):
@@ -239,3 +291,95 @@ class RemoteZipStream(io.RawIOBase):
         self._buffer = self._buffer[n:]
 
         return ret
+
+
+ ### script start
+
+class StreamProgress:
+    def __init__(self, fp, name='', total=0):
+        self.name = name
+        self.fp = fp
+        self.total = total
+        self.start_time = time.time()
+        self.last_update = 0
+        self.amtread = 0
+
+    def read(self, n):
+        r = self.fp.read(n)
+        self.amtread += len(r)
+        now = time.time()
+        if now - self.last_update > 0.1:
+            self.last_update = now
+
+            elapsed_s = now - self.start_time
+            sys.stderr.write(f'\r{elapsed_s:.0f}s  {self.amtread/10**6:.02f}/{self.total/10**6:.02f}MB  ({self.amtread/10**6/elapsed_s:.02f} MB/s)  {self.name}')
+
+        if not r:
+            sys.stderr.write('\n')
+
+        return r
+
+
+def list_files(rzf):
+    def safelog(x):
+        return 1 if x == 0 else math.ceil(math.log10(x))
+
+    digits_compr = max(safelog(f.compress_size) for f in rzf.infolist())
+    digits_plain = max(safelog(f.file_size    ) for f in rzf.infolist())
+    fmtstr = f'%{digits_compr}d -> %{digits_plain}d\t%s'
+    for f in rzf.infolist():
+        print(fmtstr % (f.compress_size, f.file_size, f.filename), file=sys.stderr)
+
+
+def extract_one(outfile, rzf, f, ofname):
+    print(f'Extracting {f.filename} to {ofname}...', file=sys.stderr)
+
+    fp = StreamProgress(rzf.open(f), name=f.filename, total=f.compress_size)
+    while r := fp.read(2**18):
+        outfile.write(r)
+
+
+def download_file(f, rzf, args):
+    if not any(fnmatch.fnmatch(f.filename, g) for g in args.files):
+        return
+
+    if args.stdout:
+        extract_one(sys.stdout.buffer, rzf, f, "stdout")
+    else:
+        path = pathlib.Path(f.filename)
+        if args.full_filepaths:
+            path.parent.mkdir(parents=True, exist_ok=True)
+        else:
+            path = path.name
+
+        with open(str(path), 'wb') as of:
+            extract_one(of, rzf, f, str(path))
+
+
+def main():
+    parser = argparse.ArgumentParser(prog='unzip-http', \
+        description="Extract individual files from .zip files over http without downloading the entire archive. HTTP server must send `Accept-Ranges: bytes` and `Content-Length` in headers.")
+
+    parser.add_argument('-l', '--list', action='store_true', default=False,
+                        help="List files in the remote zip file")
+    parser.add_argument('-f', '--full-filepaths', action='store_true', default=False,
+                        help="Recreate folder structure from zip file when extracting (instead of extracting the files to the current directory)")
+    parser.add_argument('-o', '--stdout', action='store_true', default=False,
+                        help="Write files to stdout (if multiple files: concatenate them to stdout, in zipfile order)")
+
+    parser.add_argument("url", nargs=1, help="URL of the remote zip file")
+    parser.add_argument("files", nargs='*', help="Files to extract. If no filenames given, displays .zip contents (filenames and sizes). Each filename can be a wildcard glob.")
+
+    args = parser.parse_args()
+
+    rzf = RemoteZipFile(args.url[0])
+    if args.list or len(args.files) == 0:
+        list_files(rzf)
+    else:
+        for f in rzf.infolist():
+            download_file(f, rzf, args)
+
+
+
+if __name__ == '__main__':
+    main()

unzip-http-0.4/PKG-INFO

@@ -1,45 +0,0 @@
-Metadata-Version: 2.1
-Name: unzip-http
-Version: 0.4
-Summary: extract files from .zip files over http without downloading entire archive
-Home-page: https://github.com/saulpw/unzip-http
-Author: Saul Pwanson
-License: UNKNOWN
-Description: # unzip-http
-        
-        Extract individual files from .zip files over http without downloading the entire archive.
-        
-        ## Install
-        
-            pip install unzip-http
-        
-        ## Usage
-        
-            unzip-http <url.zip> <filenames..>
-        
-        Extract <filenames> from a remote .zip at `<url>` to stdout.
-        
-        If no filenames given, displays .zip contents (filenames and sizes).
-        
-        Each filename can be a wildcard glob; all matching files are concatenated and sent to stdout in zipfile order.
-        
-        Note: HTTP server must send `Accept-Ranges: bytes` and `Content-Length` in headers.
-        
-        # Python module `unzip_http`
-        
-            import unzip_http
-        
-            rzf = unzip_http.RemoteZipFile('https://example.com/foo.zip')
-            binfp = rzf.open('bar.bin')
-            txtfp = rzf.open_text('baz.txt')
-        
-        # Credits
-        
-        `unzip-http` was written by [Saul Pwanson](https://saul.pw) and made available for use under the MIT License.
-        
-Keywords: http zip unzip
-Platform: UNKNOWN
-Classifier: Development Status :: 4 - Beta
-Classifier: Programming Language :: Python :: 3
-Requires-Python: >=3.8
-Description-Content-Type: text/markdown

unzip-http-0.4/unzip-http

@@ -1,62 +0,0 @@
-#!/usr/bin/env python3
-
-'''
-Usage:
-    unzip-http <url.zip> <filenames..>
-
-Extract <filenames> from a remote .zip at <url> to stdout.
-If no filenames given, displays .zip contents (filenames and sizes).
-Each filename can be a wildcard glob; all matching files are concatenated and sent to stdout in zipfile order.
-
-HTTP server must send `Accept-Ranges: bytes` and `Content-Length` in headers.
-'''
-
-import sys
-import io
-import time
-import fnmatch
-
-import unzip_http
-
-
-class StreamProgress:
-    def __init__(self, fp, name='', total=0):
-        self.name = name
-        self.fp = fp
-        self.total = total
-        self.start_time = time.time()
-        self.last_update = 0
-        self.amtread = 0
-
-    def read(self, n):
-        r = self.fp.read(n)
-        self.amtread += len(r)
-        now = time.time()
-        if now - self.last_update > 0.1:
-            self.last_update = now
-
-            elapsed_s = now - self.start_time
-            sys.stderr.write(f'\r{elapsed_s:.0f}s  {self.amtread/10**6:.02f}/{self.total/10**6:.02f}MB  ({self.amtread/10**6/elapsed_s:.02f} MB/s)  {self.name}')
-
-        if not r:
-            sys.stderr.write('\n')
-
-        return r
-
-
-def main(url, *globs):
-    rzf = unzip_http.RemoteZipFile(url)
-    for f in rzf.infolist():
-        if not globs:
-            print(f'{f.compress_size} -> {f.file_size}  {f.filename}')
-        elif any(fnmatch.fnmatch(f.filename, g) for g in globs):
-            fp = StreamProgress(rzf.open(f), name=f.filename, total=f.compress_size)
-            while r := fp.read(2**18):
-                sys.stdout.buffer.write(r)
-
-
-args = sys.argv[1:]
-if not args:
-    print(__doc__, file=sys.stderr)
-else:
-    main(*args)

unzip-http-0.4/unzip_http.egg-info/PKG-INFO

@@ -1,45 +0,0 @@
-Metadata-Version: 2.1
-Name: unzip-http
-Version: 0.4
-Summary: extract files from .zip files over http without downloading entire archive
-Home-page: https://github.com/saulpw/unzip-http
-Author: Saul Pwanson
-License: UNKNOWN
-Description: # unzip-http
-        
-        Extract individual files from .zip files over http without downloading the entire archive.
-        
-        ## Install
-        
-            pip install unzip-http
-        
-        ## Usage
-        
-            unzip-http <url.zip> <filenames..>
-        
-        Extract <filenames> from a remote .zip at `<url>` to stdout.
-        
-        If no filenames given, displays .zip contents (filenames and sizes).
-        
-        Each filename can be a wildcard glob; all matching files are concatenated and sent to stdout in zipfile order.
-        
-        Note: HTTP server must send `Accept-Ranges: bytes` and `Content-Length` in headers.
-        
-        # Python module `unzip_http`
-        
-            import unzip_http
-        
-            rzf = unzip_http.RemoteZipFile('https://example.com/foo.zip')
-            binfp = rzf.open('bar.bin')
-            txtfp = rzf.open_text('baz.txt')
-        
-        # Credits
-        
-        `unzip-http` was written by [Saul Pwanson](https://saul.pw) and made available for use under the MIT License.
-        
-Keywords: http zip unzip
-Platform: UNKNOWN
-Classifier: Development Status :: 4 - Beta
-Classifier: Programming Language :: Python :: 3
-Requires-Python: >=3.8
-Description-Content-Type: text/markdown