pywaybackup 1.2.0__tar.gz → 1.2.2__tar.gz

{pywaybackup-1.2.0 → pywaybackup-1.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pywaybackup
-Version: 1.2.0
+Version: 1.2.2
 Summary: Download snapshots from the Wayback Machine
 Home-page: https://github.com/bitdruid/python-wayback-machine-downloader
 Author: bitdruid

pywaybackup-1.2.2/pywaybackup/__version__.py

@@ -0,0 +1 @@
+__version__ = "1.2.2"

{pywaybackup-1.2.0 → pywaybackup-1.2.2}/pywaybackup/archive.py

@@ -11,6 +11,8 @@ import http.client
 from urllib.parse import urljoin
 from datetime import datetime, timezone
 
+from socket import timeout
+
 from pywaybackup.helper import url_get_timestamp, url_split, move_index, sanitize_filename
 
 from pywaybackup.SnapshotCollection import SnapshotCollection as sc
@@ -130,7 +132,9 @@ def query_list(url: str, range: int, start: int, end: int, explicit: bool, mode:
         try:
             cdxResult = requests.get(cdxQuery).text
         except requests.exceptions.ConnectionError as e:
-            ex.exception("Could not query snapshots", e)
+            vb.write("\nCONNECTION REFUSED -> could not query cdx server (max retries exceeded)\n")
+            os._exit(1)
+            #ex.exception("Could not query snapshots", e)
         
         if cdxbackup:
             os.makedirs(cdxbackup, exist_ok=True)
@@ -220,11 +224,13 @@ def download_loop(snapshot_queue, output, worker, retry, no_redirect, skipset=No
                 vb.write(progress=1)
         if failed_urls:
             if not attempt > max_attempt: 
+                attempt += 1
                 vb.write(f"\n-----> Worker: {worker} - Retry Timeout: 15 seconds")
                 time.sleep(15)
-            download_loop(failed_urls, output, worker, retry, no_redirect, skipset, attempt, connection, failed_urls)
+            download_loop(snapshot_queue, output, worker, retry, no_redirect, skipset, attempt, connection, failed_urls)
     except Exception as e:
         ex.exception(f"Worker: {worker} - Exception", e)
+        snapshot_queue.put(snapshot) # requeue snapshot if worker crashes
 
 
 
@@ -293,7 +299,6 @@ def download(output, snapshot_entry, connection, status_message, no_redirect=Fal
                         f"FILENAME TOO LONG -> HTTP: {response_status} - {response_status_message}\n" + \
                         f"                  -> URL: {download_url}"
                     vb.write(status_message)
-                    #skip_write(skipset, snapshot_entry["url_archive"]) if skipset is not None else None
                     return True
 
                 if not os.path.isfile(output_file):
@@ -314,7 +319,6 @@ def download(output, snapshot_entry, connection, status_message, no_redirect=Fal
                     f"           -> FILE: {output_file}"
                 vb.write(status_message)
                 sc.snapshot_entry_modify(snapshot_entry, "file", output_file)
-                #skip_write(skipset, snapshot_entry["url_archive"], output_file) if skipset is not None else None
                 return True
             
             else:
@@ -330,20 +334,8 @@ def download(output, snapshot_entry, connection, status_message, no_redirect=Fal
                 f"          -> {e}"
             vb.write(status_message)
             return False
-        # connection timeout waits and retries
-        except requests.exceptions.Timeout as e:
-            status_message = f"{status_message}\n" + \
-                f"TIMEOUT   -> ({i+1}/{max_retries}), reconnect in {sleep_time} seconds...\n" + \
-                f"         -> {e}"
-            vb.write(status_message)
-            time.sleep(sleep_time)
-        # connection refused waits and retries
-        except ConnectionRefusedError as e:
-            status_message = f"{status_message}\n" + \
-                f"REFUSED  -> ({i+1}/{max_retries}), reconnect in {sleep_time} seconds...\n" + \
-                f"         -> {e}"
-            vb.write(status_message)
-            time.sleep(sleep_time)
+        except (timeout, ConnectionRefusedError, ConnectionResetError) as e:
+            download_exception(type, e, i, max_retries, sleep_time, status_message)
     vb.write(f"FAILED  -> download, append to failed_urls: {download_url}")
     return False
 
@@ -358,6 +350,16 @@ RESPONSE_CODE_DICT = {
     503: "Service Unavailable"
 }
 
+def download_exception(type, e, i, max_retries, sleep_time, status_message):
+    """
+    Handle exceptions during the download process.
+    """
+    type = e.__class__.__name__.upper()
+    status_message = f"{status_message}\n" + \
+        f"{type} -> ({i+1}/{max_retries}), reconnect in {sleep_time} seconds...\n" + \
+    vb.write(status_message)
+    time.sleep(sleep_time)
+
 def parse_response_code(response_code: int):
     """
     Parse the response code of the Wayback Machine and return a human-readable message.
@@ -384,20 +386,21 @@ def csv_close(csv_path: str, url: str):
                 with open(csv_path, mode='a') as file: # append new rows
                     row = csv.DictWriter(file, sc.SNAPSHOT_COLLECTION[0].keys())
                     for snapshot in sc.SNAPSHOT_COLLECTION:
-                        if snapshot["url_archive"] not in existing_rows:
+                        if snapshot["response"] is not False and snapshot["url_archive"] not in existing_rows: # only append handled snapshots
                             row.writerow(snapshot)
             else: # create new file
                 with open(csv_path, mode='w') as file:
                     row = csv.DictWriter(file, sc.SNAPSHOT_COLLECTION[0].keys())
                     row.writeheader()
                     for snapshot in sc.SNAPSHOT_COLLECTION:
-                        row.writerow(snapshot)
+                        if snapshot["response"] is not False: # only append handled snapshots
+                            row.writerow(snapshot)
     except Exception as e:
         ex.exception("Could not save CSV-file", e)
 
 def csv_read(csv_file: object) -> list:
     """
-    Read the CSV file and return a list of existing snapshot urls.
+    Read the CSV file and return a list of existing snapshot urls with a status (any status means file was handled)
     """
     try:
         csv_reader = csv.reader(csv_file)

{pywaybackup-1.2.0 → pywaybackup-1.2.2}/pywaybackup.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pywaybackup
-Version: 1.2.0
+Version: 1.2.2
 Summary: Download snapshots from the Wayback Machine
 Home-page: https://github.com/bitdruid/python-wayback-machine-downloader
 Author: bitdruid

pywaybackup-1.2.0/pywaybackup/__version__.py

@@ -1 +0,0 @@
-__version__ = "1.2.0"