pywaybackup 4.2.2__tar.gz → 4.2.3__tar.gz

{pywaybackup-4.2.2/pywaybackup.egg-info → pywaybackup-4.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pywaybackup
-Version: 4.2.2
+Version: 4.2.3
 Summary: Query and download archive.org as simple as possible.
 Author-email: bitdruid <bitdruid@outlook.com>
 License: MIT License

{pywaybackup-4.2.2 → pywaybackup-4.2.3}/pyproject.toml

@@ -16,7 +16,7 @@ dependencies = [
     "python-magic-standalone==0.4.28",
 ]
 name = "pywaybackup"
-version = "4.2.2"
+version = "4.2.3"
 description = "Query and download archive.org as simple as possible."
 authors = [{ name = "bitdruid", email = "bitdruid@outlook.com" }]
 license = { file = "LICENSE" }

{pywaybackup-4.2.2 → pywaybackup-4.2.3}/pywaybackup/archive_download.py

@@ -11,7 +11,7 @@ from socket import timeout
 from urllib.parse import urljoin
 
 from pywaybackup.Exception import Exception as ex
-from pywaybackup.helper import check_nt, move_index, url_get_timestamp
+from pywaybackup.helper import add_html_extension, check_nt, move_index, url_get_timestamp
 from pywaybackup.SnapshotCollection import SnapshotCollection
 from pywaybackup.Verbosity import Verbosity as vb
 from pywaybackup.Worker import Worker
@@ -81,7 +81,17 @@ class DownloadArchive:
         sc (SnapshotCollection): The snapshot collection being processed.
     """
 
-    def __init__(self, mode: str, output: str, retry: int, no_redirect: bool, delay: int, wait: int, workers: int, merge_www: bool = True):
+    def __init__(
+        self,
+        mode: str,
+        output: str,
+        retry: int,
+        no_redirect: bool,
+        delay: int,
+        wait: int,
+        workers: int,
+        merge_www: bool = True,
+    ):
         """
         Initialize the download manager with configuration options.
 
@@ -270,7 +280,7 @@ class DownloadArchive:
         except Exception as e:
             ex.exception(f"\nWorker: {worker.id} - Exception", e)
         finally:
-            worker.close()  
+            worker.close()
 
     def _download(self, worker: Worker):
         """
@@ -291,6 +301,7 @@ class DownloadArchive:
 
         if context.response_status == 200:
             context.output_file = worker.snapshot.create_output()
+            context.output_file = add_html_extension(context.output_file, context.response_data)
             context.output_path = os.path.dirname(context.output_file)
 
             # if output_file is too long for windows, skip download
@@ -305,15 +316,6 @@ class DownloadArchive:
             # download file if not existing
             if not os.path.isfile(context.output_file):
                 with open(context.output_file, "wb") as file:
-                    if context.response.getheader("Content-Encoding") == "gzip":
-                        try:
-                            context.response_data = gzip.decompress(context.response_data)
-                        except BadGzipFile:
-                            vb.write(
-                                verbose=None,
-                                content=f"Worker: {worker.id} - GZIP DECOMPRESS SKIPPED - {context.snapshot_url}",
-                            )
-                            pass
                     file.write(context.response_data)
 
                 # check if file is downloaded
@@ -428,3 +430,13 @@ class DownloadArchive:
         context.response = worker.connection.getresponse()
         context.response_data = context.response.read()
         context.response_status = context.response.status
+
+        # decompress before sniff
+        if context.response.getheader("Content-Encoding") == "gzip":
+            try:
+                context.response_data = gzip.decompress(context.response_data)
+            except BadGzipFile:
+                vb.write(
+                    verbose=None,
+                    content=f"Worker: {worker.id} - GZIP DECOMPRESS SKIPPED - {context.snapshot_url}",
+                )

{pywaybackup-4.2.2 → pywaybackup-4.2.3}/pywaybackup/helper.py

@@ -2,6 +2,12 @@ import os
 import shutil
 import magic
 
+# one instance for the run to keep resource usage low
+_mime = magic.Magic(mime=True)
+
+# only keep the header for libmagic
+_MIME_SNIFF_BYTES = 2048
+
 
 def check_nt():
     """
@@ -128,8 +134,23 @@ def move_index(existpath: str = None, existfile: str = None, filebuffer: bytes =
 
 
 def check_index_mime(filebuffer: bytes) -> bool:
-    mime = magic.Magic(mime=True)
-    mime_type = mime.from_buffer(filebuffer)
+    mime_type = _mime.from_buffer(filebuffer[:_MIME_SNIFF_BYTES])
     if mime_type != "text/html":
         return False
     return True
+
+
+def add_html_extension(filepath: str, filebuffer: bytes) -> str:
+    """
+    Append `.html` to a file without extension if its content is html.
+
+    Urls like `/about` or `/docs/guide` carry no extension, so the snapshot is
+    written as an extensionless file that no browser or file manager opens.
+    The content type is sniffed from the buffer instead of the cdx mimetype
+    column, which is often `warc/revisit` or `unk` rather than a real type.
+    """
+    if os.path.splitext(filepath)[1]:
+        return filepath
+    if not check_index_mime(filebuffer):
+        return filepath
+    return filepath + ".html"

{pywaybackup-4.2.2 → pywaybackup-4.2.3/pywaybackup.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pywaybackup
-Version: 4.2.2
+Version: 4.2.3
 Summary: Query and download archive.org as simple as possible.
 Author-email: bitdruid <bitdruid@outlook.com>
 License: MIT License