pywaybackup 4.1.1__tar.gz → 4.1.2__tar.gz

{pywaybackup-4.1.1/pywaybackup.egg-info → pywaybackup-4.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pywaybackup
-Version: 4.1.1
+Version: 4.1.2
 Summary: Query and download archive.org as simple as possible.
 Author-email: bitdruid <bitdruid@outlook.com>
 License: MIT License
@@ -32,8 +32,9 @@ License-File: LICENSE
 Requires-Dist: SQLAlchemy==2.0.43
 Requires-Dist: requests==2.32.3
 Requires-Dist: tqdm==4.67.1
+Requires-Dist: pylibmagic==0.5.0
 Requires-Dist: python-magic==0.4.27; sys_platform == "linux"
-Requires-Dist: python-magic-bin==0.4.14; sys_platform == "win32"
+Requires-Dist: python-magic-bin==0.4.14; sys_platform == "win32" or sys_platform == "darwin"
 
 # python wayback machine downloader
 

{pywaybackup-4.1.1 → pywaybackup-4.1.2}/pyproject.toml

@@ -7,7 +7,7 @@ packages = ["pywaybackup"]
 
 [project]
 name = "pywaybackup"
-version = "4.1.1"
+version = "4.1.2"
 description = "Query and download archive.org as simple as possible."
 authors = [
     { name = "bitdruid", email = "bitdruid@outlook.com" }
@@ -19,8 +19,9 @@ dependencies = [
     "SQLAlchemy==2.0.43",
     "requests==2.32.3",
     "tqdm==4.67.1",
+    "pylibmagic==0.5.0",
     "python-magic==0.4.27; sys_platform == 'linux'",
-    "python-magic-bin==0.4.14; sys_platform == 'win32'",
+    "python-magic-bin==0.4.14; sys_platform == 'win32' or sys_platform == 'darwin'",
 ]
 
 [project.scripts]

{pywaybackup-4.1.1 → pywaybackup-4.1.2}/pywaybackup/Exception.py

@@ -36,10 +36,7 @@ class Exception:
             codeline = linecache.getline(filename, tb_line).strip()
             local_vars = tb_frame.f_locals
             exception_message += (
-                f"!-- File: {filename}\n"
-                f"!-- Function: {func_name}\n"
-                f"!-- Line: {tb_line}\n"
-                f"!-- Segment: {codeline}\n"
+                f"!-- File: {filename}\n!-- Function: {func_name}\n!-- Line: {tb_line}\n!-- Segment: {codeline}\n"
             )
         else:
             exception_message += "!-- Traceback is None\n"
@@ -96,4 +93,4 @@ class Exception:
         if issubclass(exception_type, KeyboardInterrupt):
             sys.__excepthook__(exception_type, exception, traceback)
             return
-        Exception.exception('UNCAUGHT EXCEPTION', exception, traceback)  # uncaught exceptions also with custom scheme
+        Exception.exception("UNCAUGHT EXCEPTION", exception, traceback)  # uncaught exceptions also with custom scheme

{pywaybackup-4.1.1 → pywaybackup-4.1.2}/pywaybackup/PyWayBackup.py

@@ -407,7 +407,11 @@ class PyWayBackup:
             "log": self._log,
             "debug": self._debug,
         }
-        return {key: (os.path.relpath(path) if rel else path) for key, path in files.items() if path and os.path.exists(path)}
+        return {
+            key: (os.path.relpath(path) if rel else path)
+            for key, path in files.items()
+            if path and os.path.exists(path)
+        }
 
     def status(self) -> dict:
         """

{pywaybackup-4.1.1 → pywaybackup-4.1.2}/pywaybackup/Snapshot.py

@@ -101,7 +101,9 @@ class Snapshot:
             value: New value to set for the column.
         """
         column = getattr(waybackup_snapshots, column)
-        self._db.session.execute(update(waybackup_snapshots).where(waybackup_snapshots.scid == self.scid).values({column: value}))
+        self._db.session.execute(
+            update(waybackup_snapshots).where(waybackup_snapshots.scid == self.scid).values({column: value})
+        )
         self._db.session.commit()
 
     def create_output(self):

{pywaybackup-4.1.1 → pywaybackup-4.1.2}/pywaybackup/SnapshotCollection.py

@@ -50,7 +50,9 @@ class SnapshotCollection:
 
     def _reset_locked_snapshots(self):
         """Reset locked snapshots to unprocessed in the database."""
-        self.db.session.execute(update(waybackup_snapshots).where(waybackup_snapshots.response == "LOCK").values(response=None))
+        self.db.session.execute(
+            update(waybackup_snapshots).where(waybackup_snapshots.response == "LOCK").values(response=None)
+        )
         self.db.session.commit()
 
     def _finalize_db(self):
@@ -140,11 +142,19 @@ class SnapshotCollection:
                     waybackup_snapshots.url_origin,
                     waybackup_snapshots.url_archive,
                 )
-                .filter(tuple_(waybackup_snapshots.timestamp, waybackup_snapshots.url_origin, waybackup_snapshots.url_archive).in_(keys))
+                .filter(
+                    tuple_(
+                        waybackup_snapshots.timestamp, waybackup_snapshots.url_origin, waybackup_snapshots.url_archive
+                    ).in_(keys)
+                )
                 .all()
             )
             existing_rows = set(existing)
-            new_rows = [row for row in unique_batch if (row["timestamp"], row["url_origin"], row["url_archive"]) not in existing_rows]
+            new_rows = [
+                row
+                for row in unique_batch
+                if (row["timestamp"], row["url_origin"], row["url_archive"]) not in existing_rows
+            ]
             if new_rows:
                 self.db.session.bulk_insert_mappings(waybackup_snapshots, new_rows)
                 self.db.session.commit()
@@ -200,17 +210,25 @@ class SnapshotCollection:
         # index for filtering last snapshots
         if self._mode_last:
             idx1 = Index(
-                "idx_waybackup_snapshots_url_origin_timestamp_desc", waybackup_snapshots.url_origin, waybackup_snapshots.timestamp.desc()
+                "idx_waybackup_snapshots_url_origin_timestamp_desc",
+                waybackup_snapshots.url_origin,
+                waybackup_snapshots.timestamp.desc(),
             )
             idx1.create(self.db.session.bind, checkfirst=True)
         # index for filtering first snapshots
         if self._mode_first:
             idx2 = Index(
-                "idx_waybackup_snapshots_url_origin_timestamp_asc", waybackup_snapshots.url_origin, waybackup_snapshots.timestamp.asc()
+                "idx_waybackup_snapshots_url_origin_timestamp_asc",
+                waybackup_snapshots.url_origin,
+                waybackup_snapshots.timestamp.asc(),
             )
             idx2.create(self.db.session.bind, checkfirst=True)
         # index for skippable snapshots
-        idx3 = Index("idx_waybackup_snapshots_timestamp_url_origin_response", waybackup_snapshots.timestamp, waybackup_snapshots.url_origin)
+        idx3 = Index(
+            "idx_waybackup_snapshots_timestamp_url_origin_response",
+            waybackup_snapshots.timestamp,
+            waybackup_snapshots.url_origin,
+        )
         idx3.create(self.db.session.bind, checkfirst=True)
 
     def _filter_snapshots(self):
@@ -224,7 +242,9 @@ class SnapshotCollection:
         def _filter_mode():
             self._filter_mode = 0
             if self._mode_last or self._mode_first:
-                ordering = waybackup_snapshots.timestamp.desc() if self._mode_last else waybackup_snapshots.timestamp.asc()
+                ordering = (
+                    waybackup_snapshots.timestamp.desc() if self._mode_last else waybackup_snapshots.timestamp.asc()
+                )
                 # assign row numbers per url_origin
                 rownum = (
                     func.row_number()
@@ -266,7 +286,9 @@ class SnapshotCollection:
 
         _filter_mode()
         _enumerate_counter()
-        self._filter_response = self.db.session.query(waybackup_snapshots).where(waybackup_snapshots.response.in_(["404", "301"])).count()
+        self._filter_response = (
+            self.db.session.query(waybackup_snapshots).where(waybackup_snapshots.response.in_(["404", "301"])).count()
+        )
         self.db.session.commit()
 
     def _skip_set(self):
@@ -280,7 +302,12 @@ class SnapshotCollection:
             for row in f:
                 self.db.session.execute(
                     update(waybackup_snapshots)
-                    .where(and_(waybackup_snapshots.timestamp == row["timestamp"], waybackup_snapshots.url_origin == row["url_origin"]))
+                    .where(
+                        and_(
+                            waybackup_snapshots.timestamp == row["timestamp"],
+                            waybackup_snapshots.url_origin == row["url_origin"],
+                        )
+                    )
                     .values(
                         url_archive=row["url_archive"],
                         redirect_url=row["redirect_url"],

{pywaybackup-4.1.1 → pywaybackup-4.1.2}/pywaybackup/Verbosity.py

@@ -68,9 +68,10 @@ class Verbosity:
                     cls.pbar = Progressbar(
                         unit=" snapshot",
                         desc="download file".ljust(15),
-                        total=maxval, ascii="░▒█",
-                        bar_format="{l_bar}{bar:50}{r_bar}{bar:-10b}"
-                        )
+                        total=maxval,
+                        ascii="░▒█",
+                        bar_format="{l_bar}{bar:50}{r_bar}{bar:-10b}",
+                    )
                 if cls.pbar is not None and progress is not None and progress > 0:
                     cls.pbar.update(progress)
 
@@ -93,7 +94,15 @@ class Verbosity:
 
 
 class Progressbar(Verbosity):
-    def __init__(self, unit: str, desc: str, unit_scale: bool = False, total: int = None, ascii: str = None, bar_format: str = None):
+    def __init__(
+        self,
+        unit: str,
+        desc: str,
+        unit_scale: bool = False,
+        total: int = None,
+        ascii: str = None,
+        bar_format: str = None,
+    ):
         if not super().silent:
             self.unit = unit
             self.desc = desc
@@ -101,7 +110,14 @@ class Progressbar(Verbosity):
             self.total = total
             self.ascii = ascii
             self.bar_format = bar_format
-            self.pbar = tqdm(unit=self.unit, desc=self.desc, unit_scale=self.unit_scale, total=self.total, ascii=self.ascii, bar_format=self.bar_format)
+            self.pbar = tqdm(
+                unit=self.unit,
+                desc=self.desc,
+                unit_scale=self.unit_scale,
+                total=self.total,
+                ascii=self.ascii,
+                bar_format=self.bar_format,
+            )
 
     def update(self, progress: int):
         """

{pywaybackup-4.1.1 → pywaybackup-4.1.2}/pywaybackup/archive_download.py

@@ -169,7 +169,13 @@ class DownloadArchive:
                         try:
                             download_status = self._download(worker=worker)
 
-                        except (timeout, ConnectionRefusedError, ConnectionResetError, http.client.HTTPException, Exception) as e:
+                        except (
+                            timeout,
+                            ConnectionRefusedError,
+                            ConnectionResetError,
+                            http.client.HTTPException,
+                            Exception,
+                        ) as e:
                             if isinstance(e, (timeout, ConnectionRefusedError, ConnectionResetError)):
                                 if download_attempt < download_max_attempt:
                                     download_attempt += 1  # try again 2x with same connection
@@ -293,7 +299,10 @@ class DownloadArchive:
                         try:
                             context.response_data = gzip.decompress(context.response_data)
                         except BadGzipFile:
-                            vb.write(verbose=None, content=f"Worker: {worker.id} - GZIP DECOMPRESS SKIPPED - {context.snapshot_url}")
+                            vb.write(
+                                verbose=None,
+                                content=f"Worker: {worker.id} - GZIP DECOMPRESS SKIPPED - {context.snapshot_url}",
+                            )
                             pass
                     file.write(context.response_data)
 
@@ -313,7 +322,9 @@ class DownloadArchive:
             context (DownloadContext): The download context.
             worker (Worker): The worker instance.
         """
-        worker.message.store(verbose=True, result="REDIRECT", content=f"{context.response_status} {context.response_status_message}")
+        worker.message.store(
+            verbose=True, result="REDIRECT", content=f"{context.response_status} {context.response_status_message}"
+        )
         worker.message.store(verbose=True, result="", info="FROM", content=context.snapshot_url)
         for _ in range(5):
             self.__download_response(context=context, worker=worker)
@@ -369,7 +380,9 @@ class DownloadArchive:
         Returns:
             bool: Always True (indicates result was processed).
         """
-        worker.message.store(verbose=True, result=result, content=f"{context.response_status} {context.response_status_message}")
+        worker.message.store(
+            verbose=True, result=result, content=f"{context.response_status} {context.response_status_message}"
+        )
         worker.message.store(verbose=False, result=result)
         worker.message.store(verbose=True, result="", info="URL", content=context.snapshot_url)
         worker.message.store(verbose=True, result="", info="FILE", content=context.output_file)
@@ -386,7 +399,9 @@ class DownloadArchive:
         Returns:
             bool: Always False (indicates failure was processed).
         """
-        worker.message.store(verbose=None, result="UNKNOWN", content=f"{context.response_status} {context.response_status_message}")
+        worker.message.store(
+            verbose=None, result="UNKNOWN", content=f"{context.response_status} {context.response_status_message}"
+        )
         worker.message.store(verbose=True, result="", info="URL", content=context.snapshot_url)
         return False
 

{pywaybackup-4.1.1 → pywaybackup-4.1.2}/pywaybackup/archive_save.py

@@ -34,10 +34,13 @@ def save_page(url: str):
 
         if response_status == 302:
             location = response.getheader("Location")
-            snapshot_timestamp = datetime.strptime(url_get_timestamp(location), "%Y%m%d%H%M%S").strftime("%Y-%m-%d %H:%M:%S")
+            snapshot_timestamp = datetime.strptime(url_get_timestamp(location), "%Y%m%d%H%M%S").strftime(
+                "%Y-%m-%d %H:%M:%S"
+            )
             current_timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
             timestamp_difference = (
-                datetime.strptime(current_timestamp, "%Y-%m-%d %H:%M:%S") - datetime.strptime(snapshot_timestamp, "%Y-%m-%d %H:%M:%S")
+                datetime.strptime(current_timestamp, "%Y-%m-%d %H:%M:%S")
+                - datetime.strptime(snapshot_timestamp, "%Y-%m-%d %H:%M:%S")
             ).seconds / 60
             timestamp_difference = int(round(timestamp_difference, 0))
 
@@ -45,7 +48,10 @@ def save_page(url: str):
                 vb.write(verbose=None, content="\n-----> Response: 302 (new snapshot)")
                 vb.write(verbose=None, content=f"SNAPSHOT URL: {location}")
             elif timestamp_difference >= 1:
-                vb.write(verbose=None, content=f"\n-----> Response: 302 (existing snapshot - wait for {60 - timestamp_difference} minutes)")
+                vb.write(
+                    verbose=None,
+                    content=f"\n-----> Response: 302 (existing snapshot - wait for {60 - timestamp_difference} minutes)",
+                )
                 vb.write(verbose=None, content=f"SNAPSHOT URL: {location}")
                 vb.write(verbose=None, content=f"WAYBACK TIME: {snapshot_timestamp}")
                 vb.write(verbose=None, content=f"REQUEST TIME: {current_timestamp}")

{pywaybackup-4.1.1 → pywaybackup-4.1.2}/pywaybackup/db.py

@@ -17,7 +17,7 @@ from sqlalchemy import (
 )
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import sessionmaker
-from typing import Optional # python 3.8
+from typing import Optional  # python 3.8
 
 Base = declarative_base()
 
@@ -113,7 +113,9 @@ class Database:
         Base.metadata.create_all(engine)
 
         db = Database()
-        if db.session.execute(select(waybackup_job.query_identifier).where(query_identifier == query_identifier)).fetchone():
+        if db.session.execute(
+            select(waybackup_job.query_identifier).where(query_identifier == query_identifier)
+        ).fetchone():
             cls.query_exist = True
             cls.query_progress = db.get_progress()
         else:
@@ -140,7 +142,9 @@ class Database:
         """
         progress = f"{(done):,} / {(total):,}"
         self.session.execute(
-            update(waybackup_job).where(waybackup_job.query_identifier == self.query_identifier).values(query_progress=progress)
+            update(waybackup_job)
+            .where(waybackup_job.query_identifier == self.query_identifier)
+            .values(query_progress=progress)
         )
         self.session.commit()
 
@@ -180,19 +184,31 @@ class Database:
         """
         Mark the job's insertion phase as complete in the database.
         """
-        self.session.execute(update(waybackup_job).where(waybackup_job.query_identifier == self.query_identifier).values(insert_complete=1))
+        self.session.execute(
+            update(waybackup_job)
+            .where(waybackup_job.query_identifier == self.query_identifier)
+            .values(insert_complete=1)
+        )
         self.session.commit()
 
     def set_index_complete(self):
         """
         Mark the job's indexing phase as complete in the database.
         """
-        self.session.execute(update(waybackup_job).where(waybackup_job.query_identifier == self.query_identifier).values(index_complete=1))
+        self.session.execute(
+            update(waybackup_job)
+            .where(waybackup_job.query_identifier == self.query_identifier)
+            .values(index_complete=1)
+        )
         self.session.commit()
 
     def set_filter_complete(self):
         """
         Mark the job's filtering phase as complete in the database.
         """
-        self.session.execute(update(waybackup_job).where(waybackup_job.query_identifier == self.query_identifier).values(filter_complete=1))
+        self.session.execute(
+            update(waybackup_job)
+            .where(waybackup_job.query_identifier == self.query_identifier)
+            .values(filter_complete=1)
+        )
         self.session.commit()

{pywaybackup-4.1.1 → pywaybackup-4.1.2}/pywaybackup/files.py

@@ -54,7 +54,9 @@ class CDXquery:
 
         limit = f"&limit={self.limit}" if self.limit else ""
 
-        filter_statuscode = f"&filter=statuscode:({'|'.join(self.filter_statuscode)})$" if self.filter_statuscode else ""
+        filter_statuscode = (
+            f"&filter=statuscode:({'|'.join(self.filter_statuscode)})$" if self.filter_statuscode else ""
+        )
         filter_filetype = f"&filter=original:.*\\.({'|'.join(self.filter_filetype)})$" if self.filter_filetype else ""
 
         return f"https://web.archive.org/cdx/search/cdx?output=json&url={cdx_url}{period}&fl=timestamp,digest,mimetype,statuscode,original{limit}{filter_filetype}{filter_statuscode}"

{pywaybackup-4.1.1 → pywaybackup-4.1.2}/pywaybackup/helper.py

@@ -1,5 +1,6 @@
 import os
 import shutil
+import pylibmagic
 import magic
 
 
@@ -88,18 +89,14 @@ def move_index(existpath: str = None, existfile: str = None, filebuffer: bytes =
         shutil.move(existpath, existpath + "_exist")
         os.makedirs(existpath, exist_ok=True)
         if not check_index_mime(existpath):
-            new_file = os.path.join(
-                existpath, os.path.basename(os.path.normpath(existpath))
-            )
+            new_file = os.path.join(existpath, os.path.basename(os.path.normpath(existpath)))
         else:
             new_file = os.path.join(existpath, "index.html")
         shutil.move(existpath + "_exist", new_file)
     elif existfile:
         if filebuffer:
             if not check_index_mime(filebuffer):
-                return os.path.join(
-                    existfile, os.path.basename(os.path.normpath(existfile))
-                )
+                return os.path.join(existfile, os.path.basename(os.path.normpath(existfile)))
             else:
                 return os.path.join(existfile, "index.html")
 

{pywaybackup-4.1.1 → pywaybackup-4.1.2/pywaybackup.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pywaybackup
-Version: 4.1.1
+Version: 4.1.2
 Summary: Query and download archive.org as simple as possible.
 Author-email: bitdruid <bitdruid@outlook.com>
 License: MIT License
@@ -32,8 +32,9 @@ License-File: LICENSE
 Requires-Dist: SQLAlchemy==2.0.43
 Requires-Dist: requests==2.32.3
 Requires-Dist: tqdm==4.67.1
+Requires-Dist: pylibmagic==0.5.0
 Requires-Dist: python-magic==0.4.27; sys_platform == "linux"
-Requires-Dist: python-magic-bin==0.4.14; sys_platform == "win32"
+Requires-Dist: python-magic-bin==0.4.14; sys_platform == "win32" or sys_platform == "darwin"
 
 # python wayback machine downloader
 

{pywaybackup-4.1.1 → pywaybackup-4.1.2}/pywaybackup.egg-info/requires.txt

@@ -1,9 +1,10 @@
 SQLAlchemy==2.0.43
 requests==2.32.3
 tqdm==4.67.1
+pylibmagic==0.5.0
 
 [:sys_platform == "linux"]
 python-magic==0.4.27
 
-[:sys_platform == "win32"]
+[:sys_platform == "win32" or sys_platform == "darwin"]
 python-magic-bin==0.4.14