pywaybackup 1.2.1__tar.gz → 1.2.3__tar.gz

{pywaybackup-1.2.1 → pywaybackup-1.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pywaybackup
-Version: 1.2.1
+Version: 1.2.3
 Summary: Download snapshots from the Wayback Machine
 Home-page: https://github.com/bitdruid/python-wayback-machine-downloader
 Author: bitdruid

pywaybackup-1.2.3/pywaybackup/__version__.py

@@ -0,0 +1 @@
+__version__ = "1.2.3"

{pywaybackup-1.2.1 → pywaybackup-1.2.3}/pywaybackup/archive.py

@@ -11,6 +11,8 @@ import http.client
 from urllib.parse import urljoin
 from datetime import datetime, timezone
 
+from socket import timeout
+
 from pywaybackup.helper import url_get_timestamp, url_split, move_index, sanitize_filename
 
 from pywaybackup.SnapshotCollection import SnapshotCollection as sc
@@ -130,7 +132,9 @@ def query_list(url: str, range: int, start: int, end: int, explicit: bool, mode:
         try:
             cdxResult = requests.get(cdxQuery).text
         except requests.exceptions.ConnectionError as e:
-            ex.exception("Could not query snapshots", e)
+            vb.write("\nCONNECTION REFUSED -> could not query cdx server (max retries exceeded)\n")
+            os._exit(1)
+            #ex.exception("Could not query snapshots", e)
         
         if cdxbackup:
             os.makedirs(cdxbackup, exist_ok=True)
@@ -220,9 +224,10 @@ def download_loop(snapshot_queue, output, worker, retry, no_redirect, skipset=No
                 vb.write(progress=1)
         if failed_urls:
             if not attempt > max_attempt: 
+                attempt += 1
                 vb.write(f"\n-----> Worker: {worker} - Retry Timeout: 15 seconds")
                 time.sleep(15)
-            download_loop(failed_urls, output, worker, retry, no_redirect, skipset, attempt, connection, failed_urls)
+            download_loop(snapshot_queue, output, worker, retry, no_redirect, skipset, attempt, connection, failed_urls)
     except Exception as e:
         ex.exception(f"Worker: {worker} - Exception", e)
         snapshot_queue.put(snapshot) # requeue snapshot if worker crashes
@@ -329,20 +334,8 @@ def download(output, snapshot_entry, connection, status_message, no_redirect=Fal
                 f"          -> {e}"
             vb.write(status_message)
             return False
-        # connection timeout waits and retries
-        except requests.exceptions.Timeout as e:
-            status_message = f"{status_message}\n" + \
-                f"TIMEOUT   -> ({i+1}/{max_retries}), reconnect in {sleep_time} seconds...\n" + \
-                f"         -> {e}"
-            vb.write(status_message)
-            time.sleep(sleep_time)
-        # connection refused waits and retries
-        except ConnectionRefusedError as e:
-            status_message = f"{status_message}\n" + \
-                f"REFUSED  -> ({i+1}/{max_retries}), reconnect in {sleep_time} seconds...\n" + \
-                f"         -> {e}"
-            vb.write(status_message)
-            time.sleep(sleep_time)
+        except (timeout, ConnectionRefusedError, ConnectionResetError) as e:
+            download_exception(type, e, i, max_retries, sleep_time, status_message)
     vb.write(f"FAILED  -> download, append to failed_urls: {download_url}")
     return False
 
@@ -357,6 +350,16 @@ RESPONSE_CODE_DICT = {
     503: "Service Unavailable"
 }
 
+def download_exception(type, e, i, max_retries, sleep_time, status_message):
+    """
+    Handle exceptions during the download process.
+    """
+    type = e.__class__.__name__.upper()
+    status_message = f"{status_message}\n" + \
+        f"{type} -> ({i+1}/{max_retries}), reconnect in {sleep_time} seconds...\n"
+    vb.write(status_message)
+    time.sleep(sleep_time)
+
 def parse_response_code(response_code: int):
     """
     Parse the response code of the Wayback Machine and return a human-readable message.

{pywaybackup-1.2.1 → pywaybackup-1.2.3}/pywaybackup.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pywaybackup
-Version: 1.2.1
+Version: 1.2.3
 Summary: Download snapshots from the Wayback Machine
 Home-page: https://github.com/bitdruid/python-wayback-machine-downloader
 Author: bitdruid

pywaybackup-1.2.1/pywaybackup/__version__.py

@@ -1 +0,0 @@
-__version__ = "1.2.1"