pywaybackup 1.4.0__tar.gz → 1.4.2__tar.gz

{pywaybackup-1.4.0/pywaybackup.egg-info → pywaybackup-1.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pywaybackup
-Version: 1.4.0
+Version: 1.4.2
 Summary: Download snapshots from the Wayback Machine
 Home-page: https://github.com/bitdruid/python-wayback-machine-downloader
 Author: bitdruid

{pywaybackup-1.4.0 → pywaybackup-1.4.2}/pywaybackup/Verbosity.py

@@ -88,12 +88,13 @@ class Message(Verbosity):
         self.message = {}
 
     def __str__(self):
-        return self.message
+        return str(self.message)
 
     def store(self, status: str = "", type: str = "", message: str = "", level: str = "info"):
         if level not in self.message:
             self.message[level] = []
         self.message[level].append(super().generate_logline(status, type, message))
+        #super().write(message=f"Stored message: {status} -> {type}: {message}")
 
     def clear(self):
         self.message = {}

pywaybackup-1.4.2/pywaybackup/__version__.py

@@ -0,0 +1 @@
+__version__ = "1.4.2"

{pywaybackup-1.4.0 → pywaybackup-1.4.2}/pywaybackup/archive.py

@@ -263,7 +263,7 @@ def download(output, snapshot_entry, connection, status_message, no_redirect=Fal
     download_url = snapshot_entry["url_archive"]
     encoded_download_url = urllib.parse.quote(download_url, safe=':/') # used for GET - otherwise always download_url
     max_retries = 2
-    sleep_time = 45
+    sleep_time = 50
     headers = {'User-Agent': f'bitdruid-python-wayback-downloader/{__version__}'}
     success = False
     for i in range(max_retries):
@@ -325,7 +325,7 @@ def download(output, snapshot_entry, connection, status_message, no_redirect=Fal
                 status_message.store(status="UNEXPECTED", type="HTTP", message=f"{response.status} - {response_status_message}")
                 status_message.store(status="", type="URL", message=download_url)
                 status_message.write()
-                continue
+                break
         # exception handling        
         except (http.client.HTTPException, timeout, ConnectionRefusedError, ConnectionResetError) as e:
             download_exception(type, e, i, max_retries, sleep_time, status_message)
@@ -390,24 +390,26 @@ def csv_close(csv_path: str, url: str):
     try:
         csv_path = csv_filepath(csv_path, url)
         if sc.count(collection=True) > 0:
-            if os.path.exists(csv_path): # append to existing file
-                existing_rows = set()
-                with open(csv_path, mode='r', newline='') as file: # read existing rows
-                    existing_rows = set(csv_read(file))
-                with open(csv_path, mode='a', newline='') as file: # append new rows
-                    row = csv.DictWriter(file, sc.SNAPSHOT_COLLECTION[0].keys())
-                    for snapshot in sc.SNAPSHOT_COLLECTION:
-                        if snapshot["response"] is not False and snapshot["url_archive"] not in existing_rows: # only append handled snapshots
-                            row.writerow(snapshot)
-            else: # create new file
+            new_rows = [snapshot for snapshot in sc.SNAPSHOT_COLLECTION 
+                        if ("response" in snapshot and snapshot["response"] is not False and "url_archive" in snapshot) or 
+                           ("digest" in snapshot)]
+            
+            if os.path.exists(csv_path):  # append to existing file
+                existing_rows = set(csv_read(open(csv_path, mode='r', newline='')))
+                
+                with open(csv_path, mode='a', newline='') as file:  # append new rows
+                    row_writer = csv.DictWriter(file, sc.SNAPSHOT_COLLECTION[0].keys())
+                    for snapshot in new_rows:
+                        snapshot_tuple = tuple(snapshot.values())
+                        if snapshot_tuple not in existing_rows:
+                            row_writer.writerow(snapshot)
+            else:  # create new file
                 with open(csv_path, mode='w', newline='') as file:
-                    row = csv.DictWriter(file, sc.SNAPSHOT_COLLECTION[0].keys())
-                    row.writeheader()
-                    for snapshot in sc.SNAPSHOT_COLLECTION:
-                        if snapshot["response"] is not False: # only append handled snapshots
-                            row.writerow(snapshot)
+                    row_writer = csv.DictWriter(file, sc.SNAPSHOT_COLLECTION[0].keys())
+                    row_writer.writeheader()
+                    row_writer.writerows(new_rows)
     except Exception as e:
-        ex.exception("Could not save CSV-file", e)
+        ex.exception("Could not save CSV file", e)
 
 def csv_read(csv_file: object) -> list:
     """

{pywaybackup-1.4.0 → pywaybackup-1.4.2}/pywaybackup/helper.py

@@ -37,7 +37,7 @@ def url_get_timestamp(url):
         """
         Extract the timestamp from a wayback machine URL.
         """
-        timestamp = url.split("web.archive.org/web/")[1].split("/")[0]
+        timestamp = url.split("web/")[1].split("/")[0]
         if "id_" in url: timestamp = timestamp.split("id_")[0]            
         return timestamp
 

{pywaybackup-1.4.0 → pywaybackup-1.4.2/pywaybackup.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pywaybackup
-Version: 1.4.0
+Version: 1.4.2
 Summary: Download snapshots from the Wayback Machine
 Home-page: https://github.com/bitdruid/python-wayback-machine-downloader
 Author: bitdruid

pywaybackup-1.4.0/pywaybackup/__version__.py

@@ -1 +0,0 @@
-__version__ = "1.4.0"