pywaybackup 4.1.2__tar.gz → 4.1.3__tar.gz

{pywaybackup-4.1.2/pywaybackup.egg-info → pywaybackup-4.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pywaybackup
-Version: 4.1.2
+Version: 4.1.3
 Summary: Query and download archive.org as simple as possible.
 Author-email: bitdruid <bitdruid@outlook.com>
 License: MIT License
@@ -29,6 +29,7 @@ Project-URL: homepage, https://github.com/bitdruid/python-wayback-machine-downlo
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: ruff
 Requires-Dist: SQLAlchemy==2.0.43
 Requires-Dist: requests==2.32.3
 Requires-Dist: tqdm==4.67.1

{pywaybackup-4.1.2 → pywaybackup-4.1.3}/pyproject.toml

@@ -2,20 +2,14 @@
 requires = ["setuptools", "wheel"]
 build-backend = "setuptools.build_meta"
 
+
 [tool.setuptools]
 packages = ["pywaybackup"]
 
+
 [project]
-name = "pywaybackup"
-version = "4.1.2"
-description = "Query and download archive.org as simple as possible."
-authors = [
-    { name = "bitdruid", email = "bitdruid@outlook.com" }
-]
-license = { file = "LICENSE" }
-readme = "README.md"
-requires-python = ">=3.8"
 dependencies = [
+    "ruff",
     "SQLAlchemy==2.0.43",
     "requests==2.32.3",
     "tqdm==4.67.1",
@@ -23,9 +17,23 @@ dependencies = [
     "python-magic==0.4.27; sys_platform == 'linux'",
     "python-magic-bin==0.4.14; sys_platform == 'win32' or sys_platform == 'darwin'",
 ]
+name = "pywaybackup"
+version = "4.1.3"
+description = "Query and download archive.org as simple as possible."
+authors = [{ name = "bitdruid", email = "bitdruid@outlook.com" }]
+license = { file = "LICENSE" }
+readme = "README.md"
+requires-python = ">=3.8"
+
 
 [project.scripts]
 waybackup = "pywaybackup.main:cli"
 
+
 [project.urls]
-homepage = "https://github.com/bitdruid/python-wayback-machine-downloader"
+homepage = "https://github.com/bitdruid/python-wayback-machine-downloader"
+
+
+[tool.ruff]
+line-length = 120
+exclude = ["pywaybackup/Arguments.py"]

{pywaybackup-4.1.2 → pywaybackup-4.1.3}/pywaybackup/Arguments.py

@@ -40,6 +40,7 @@ class Arguments:
         behavior.add_argument("--retry", type=int, default=0, metavar="", help="retry failed downloads (opt tries as int, else infinite)")
         behavior.add_argument("--workers", type=int, default=1, metavar="", help="number of workers (simultaneous downloads)")
         behavior.add_argument("--delay", type=int, default=0, metavar="", help="delay between each download in seconds")
+        behavior.add_argument("--wait", type=int, default=15, metavar="", help="seconds to wait before renewing connection after HTTP errors or snapshot download errors (default: 15)",)
 
         special = parser.add_argument_group("special")
         special.add_argument("--reset", action="store_true", help="reset the job and ignore existing cdx/db/csv files")
@@ -55,4 +56,3 @@ class Arguments:
     def get_args(self) -> dict:
         """Returns the parsed arguments as a dictionary."""
         return vars(self.args)
-

{pywaybackup-4.1.2 → pywaybackup-4.1.3}/pywaybackup/PyWayBackup.py

@@ -9,7 +9,7 @@ import pywaybackup.archive_save as archive_save
 from pywaybackup.archive_download import DownloadArchive
 from pywaybackup.db import Database as db
 from pywaybackup.Exception import Exception as ex
-from pywaybackup.files import CDXfile, CDXquery, CSVfile, File
+from pywaybackup.files import CDXfile, CDXquery, CSVfile
 from pywaybackup.helper import sanitize_filename, url_split
 from pywaybackup.SnapshotCollection import SnapshotCollection
 from pywaybackup.Verbosity import Verbosity as vb
@@ -23,7 +23,7 @@ class _Status:
 
     Attributes:
         sc (SnapshotCollection): The current snapshot collection being processed.
-        task (str): The current task being performed (e.g., 'initializing', 'downloading cdx', 'preparing snapshots', 'downloading snapshots', 'done').
+        task (str): The current task being performed (e.g., 'initializing', 'downloading cdx', ...).
         handled (int): The number of snapshots that have been processed so far.
         total (int): The total number of snapshots to be processed.
         progress (float): The progress of the backup process as a percentage.
@@ -129,6 +129,7 @@ class PyWayBackup:
         retry: int = 0,
         workers: int = 1,
         delay: int = 0,
+        wait: int = 15,
         reset: bool = False,
         keep: bool = False,
         silent: bool = True,
@@ -156,6 +157,8 @@ class PyWayBackup:
         self._retry = retry
         self._workers = workers
         self._delay = delay
+        self._wait = wait
+
         self._reset = reset
         self._keep = keep
 
@@ -344,6 +347,7 @@ class PyWayBackup:
             retry=self._retry,
             no_redirect=self._no_redirect,
             delay=self._delay,
+            wait=self._wait,
             workers=self._workers,
         )
         downloader.run(SnapshotCollection=collection)

{pywaybackup-4.1.2 → pywaybackup-4.1.3}/pywaybackup/archive_download.py

@@ -81,7 +81,7 @@ class DownloadArchive:
         sc (SnapshotCollection): The snapshot collection being processed.
     """
 
-    def __init__(self, mode: str, output: str, retry: int, no_redirect: bool, delay: int, workers: int):
+    def __init__(self, mode: str, output: str, retry: int, no_redirect: bool, delay: int, wait: int, workers: int):
         """
         Initialize the download manager with configuration options.
 
@@ -98,8 +98,8 @@ class DownloadArchive:
         self.retry = retry
         self.no_redirect = no_redirect
         self.delay = delay
+        self.wait = wait
         self.workers = workers
-        self.no_redirect = no_redirect
         self.sc = None
 
     def run(self, SnapshotCollection: SnapshotCollection):
@@ -208,7 +208,7 @@ class DownloadArchive:
                                             f"\n-----> Worker: {worker.id}"
                                             f" - Attempt: [{worker.attempt}/{retry_max_attempt}]"
                                             f" Snapshot ID: [{worker.snapshot.counter}/{self.sc._snapshot_total}]"
-                                            f" - {e.__class__.__name__} - renewing connection in 15 seconds..."
+                                            f" - {e.__class__.__name__} - renewing connection in {self.wait * download_attempt} seconds..."
                                         ),
                                     )
                                     vb.write(
@@ -216,10 +216,10 @@ class DownloadArchive:
                                         content=(
                                             f"Worker: {worker.id}"
                                             f" - Snapshot {worker.snapshot.counter}/{self.sc._snapshot_total}"
-                                            f" - renewing connection in 15 seconds..."
+                                            f" - renewing connection in {self.wait * download_attempt} seconds..."
                                         ),
                                     )
-                                    time.sleep(15)
+                                    time.sleep(self.wait * download_attempt)
                                     worker.refresh_connection()
                                     continue
                             else:
@@ -244,9 +244,13 @@ class DownloadArchive:
 
                         # depends on user - retries after timeout or proceed to next snapshot
                         if self.retry > 0:
-                            worker.message.store(verbose=True, result="FAILED", content="retry timeout: 15 seconds...")
+                            worker.message.store(
+                                verbose=True,
+                                result="FAILED",
+                                content=f"retry timeout: {self.wait * worker.attempt} seconds...",
+                            )
                             worker.message.write()
-                            time.sleep(15)
+                            time.sleep(self.wait * worker.attempt)
                         else:
                             worker.message.store(verbose=None, result="FAILED", content="no attempt left")
                             worker.message.write()

{pywaybackup-4.1.2 → pywaybackup-4.1.3}/pywaybackup/files.py

@@ -59,7 +59,15 @@ class CDXquery:
         )
         filter_filetype = f"&filter=original:.*\\.({'|'.join(self.filter_filetype)})$" if self.filter_filetype else ""
 
-        return f"https://web.archive.org/cdx/search/cdx?output=json&url={cdx_url}{period}&fl=timestamp,digest,mimetype,statuscode,original{limit}{filter_filetype}{filter_statuscode}"
+        return (
+            f"https://web.archive.org/cdx/search/cdx?"
+            f"output=json"
+            f"&url={cdx_url}{period}"
+            f"&fl=timestamp,digest,mimetype,statuscode,original"
+            f"{limit}"
+            f"{filter_filetype}"
+            f"{filter_statuscode}"
+        )
 
 
 class File:

{pywaybackup-4.1.2 → pywaybackup-4.1.3/pywaybackup.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pywaybackup
-Version: 4.1.2
+Version: 4.1.3
 Summary: Query and download archive.org as simple as possible.
 Author-email: bitdruid <bitdruid@outlook.com>
 License: MIT License
@@ -29,6 +29,7 @@ Project-URL: homepage, https://github.com/bitdruid/python-wayback-machine-downlo
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: ruff
 Requires-Dist: SQLAlchemy==2.0.43
 Requires-Dist: requests==2.32.3
 Requires-Dist: tqdm==4.67.1

{pywaybackup-4.1.2 → pywaybackup-4.1.3}/pywaybackup.egg-info/requires.txt

@@ -1,3 +1,4 @@
+ruff
 SQLAlchemy==2.0.43
 requests==2.32.3
 tqdm==4.67.1