pywaybackup 1.2.0__tar.gz → 1.2.1__tar.gz

{pywaybackup-1.2.0 → pywaybackup-1.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pywaybackup
-Version: 1.2.0
+Version: 1.2.1
 Summary: Download snapshots from the Wayback Machine
 Home-page: https://github.com/bitdruid/python-wayback-machine-downloader
 Author: bitdruid

pywaybackup-1.2.1/pywaybackup/__version__.py

@@ -0,0 +1 @@
+__version__ = "1.2.1"

{pywaybackup-1.2.0 → pywaybackup-1.2.1}/pywaybackup/archive.py

@@ -225,6 +225,7 @@ def download_loop(snapshot_queue, output, worker, retry, no_redirect, skipset=No
             download_loop(failed_urls, output, worker, retry, no_redirect, skipset, attempt, connection, failed_urls)
     except Exception as e:
         ex.exception(f"Worker: {worker} - Exception", e)
+        snapshot_queue.put(snapshot) # requeue snapshot if worker crashes
 
 
 
@@ -293,7 +294,6 @@ def download(output, snapshot_entry, connection, status_message, no_redirect=Fal
                         f"FILENAME TOO LONG -> HTTP: {response_status} - {response_status_message}\n" + \
                         f"                  -> URL: {download_url}"
                     vb.write(status_message)
-                    #skip_write(skipset, snapshot_entry["url_archive"]) if skipset is not None else None
                     return True
 
                 if not os.path.isfile(output_file):
@@ -314,7 +314,6 @@ def download(output, snapshot_entry, connection, status_message, no_redirect=Fal
                     f"           -> FILE: {output_file}"
                 vb.write(status_message)
                 sc.snapshot_entry_modify(snapshot_entry, "file", output_file)
-                #skip_write(skipset, snapshot_entry["url_archive"], output_file) if skipset is not None else None
                 return True
             
             else:
@@ -384,20 +383,21 @@ def csv_close(csv_path: str, url: str):
                 with open(csv_path, mode='a') as file: # append new rows
                     row = csv.DictWriter(file, sc.SNAPSHOT_COLLECTION[0].keys())
                     for snapshot in sc.SNAPSHOT_COLLECTION:
-                        if snapshot["url_archive"] not in existing_rows:
+                        if snapshot["response"] is not False and snapshot["url_archive"] not in existing_rows: # only append handled snapshots
                             row.writerow(snapshot)
             else: # create new file
                 with open(csv_path, mode='w') as file:
                     row = csv.DictWriter(file, sc.SNAPSHOT_COLLECTION[0].keys())
                     row.writeheader()
                     for snapshot in sc.SNAPSHOT_COLLECTION:
-                        row.writerow(snapshot)
+                        if snapshot["response"] is not False: # only append handled snapshots
+                            row.writerow(snapshot)
     except Exception as e:
         ex.exception("Could not save CSV-file", e)
 
 def csv_read(csv_file: object) -> list:
     """
-    Read the CSV file and return a list of existing snapshot urls.
+    Read the CSV file and return a list of existing snapshot urls with a status (any status means file was handled)
     """
     try:
         csv_reader = csv.reader(csv_file)

{pywaybackup-1.2.0 → pywaybackup-1.2.1}/pywaybackup.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pywaybackup
-Version: 1.2.0
+Version: 1.2.1
 Summary: Download snapshots from the Wayback Machine
 Home-page: https://github.com/bitdruid/python-wayback-machine-downloader
 Author: bitdruid

pywaybackup-1.2.0/pywaybackup/__version__.py

@@ -1 +0,0 @@
-__version__ = "1.2.0"