pywaybackup 3.0.2__tar.gz → 3.0.4__tar.gz

{pywaybackup-3.0.2/pywaybackup.egg-info → pywaybackup-3.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: pywaybackup
-Version: 3.0.2
+Version: 3.0.4
 Summary: Query and download archive.org as simple as possible.
 Author-email: bitdruid <bitdruid@outlook.com>
 License: MIT License

{pywaybackup-3.0.2 → pywaybackup-3.0.4}/pyproject.toml

@@ -7,7 +7,7 @@ packages = ["pywaybackup"]
 
 [project]
 name = "pywaybackup"
-version = "3.0.2"
+version = "3.0.4"
 description = "Query and download archive.org as simple as possible."
 authors = [
     { name = "bitdruid", email = "bitdruid@outlook.com" }

{pywaybackup-3.0.2 → pywaybackup-3.0.4}/pywaybackup/Arguments.py

@@ -5,6 +5,7 @@ import argparse
 from importlib.metadata import version
 
 from pywaybackup.helper import url_split, sanitize_filename
+from pywaybackup.Exception import Exception as ex
 
 class Arguments:
     
@@ -84,6 +85,8 @@ class Configuration:
             cls.mode = "last"
         if cls.first:
             cls.mode = "first"
+        if cls.save:
+            cls.mode = "save"
         
         if cls.filetype:
             cls.filetype = [ft.lower().strip() for ft in cls.filetype.split(",")]

{pywaybackup-3.0.2 → pywaybackup-3.0.4}/pywaybackup/Exception.py

@@ -105,4 +105,4 @@ class Exception:
             sys.__excepthook__(exception_type, exception, traceback)
             return
         Exception.exception("UNCAUGHT EXCEPTION", exception, traceback) # uncaught exceptions also with custom scheme
-    
+    

{pywaybackup-3.0.2 → pywaybackup-3.0.4}/pywaybackup/SnapshotCollection.py

@@ -1,10 +1,15 @@
-from pywaybackup.Verbosity import Verbosity as vb
-from pywaybackup.helper import url_split
-from pywaybackup.db import Database
-from tqdm import tqdm
 import json
 import csv
 import os
+import threading
+
+from tqdm import tqdm
+
+from pywaybackup.Verbosity import Verbosity as vb
+from pywaybackup.helper import url_split
+from pywaybackup.db import Database
+
+LOCK = threading.Lock() # thread safe lock
 
 class SnapshotCollection:
     """
@@ -278,27 +283,31 @@ class SnapshotCollection:
         """
         Modify a snapshot-row in the snapshot table.
         """
-        connection.cursor.execute(
-            f"""
-            UPDATE snapshot_tbl
-            SET {column} = ?
-            WHERE rowid = ?
-            """,
-            (value, snapshot_id)
-        )
-        connection.conn.commit()
+        global LOCK
+        with LOCK:
+            connection.cursor.execute(
+                f"""
+                UPDATE snapshot_tbl
+                SET {column} = ?
+                WHERE rowid = ?
+                """,
+                (value, snapshot_id)
+            )
+            connection.conn.commit()
 
     def get_snapshot(connection):
         """
         Get a snapshot-row from the snapshot table with response NULL. (not processed)
         """
-        connection.cursor.execute(
-            """
-            SELECT rowid, * FROM snapshot_tbl WHERE response IS NULL LIMIT 1
-            """
-        )
-        row = connection.cursor.fetchone()
-        return row
+        global LOCK
+        with LOCK:
+            connection.cursor.execute(
+                """
+                SELECT rowid, * FROM snapshot_tbl WHERE response IS NULL LIMIT 1
+                """
+            )
+            row = connection.cursor.fetchone()
+            return row
 
     @classmethod
     def create_output(cls, url: str, timestamp: str, output: str):

{pywaybackup-3.0.2 → pywaybackup-3.0.4}/pywaybackup/archive.py

@@ -224,15 +224,16 @@ def download_loop(output, worker, retry, no_redirect, delay):
             
             snapshot = sc.get_snapshot(db)
             if not snapshot: break
-            sc.modify_snapshot(db, snapshot["rowid"], "response", "LOCK") # mark as locked for other workers
-            SNAPSHOT_CURRENT = sc.SNAPSHOT_HANDLED + 1
+            # mark as locked for other workers // only visual because get_snapshot fetches by NULL
+            sc.modify_snapshot(db, snapshot["rowid"], "response", "LOCK")
+            SNAPSHOT_CURRENT = snapshot["rowid"]
             
             retry_attempt = 1
             retry_max_attempt = retry if retry > 0 else retry + 1
             status_message = Message()
             
             while retry_attempt <= retry_max_attempt: # retry as given by user
-                status_message.store(message=f"\n-----> Worker: {worker} - Attempt: [{retry_attempt}/{retry_max_attempt}] Snapshot [{SNAPSHOT_CURRENT}/{sc.SNAPSHOT_TOTAL}]")
+                status_message.store(message=f"\n-----> Worker: {worker} - Attempt: [{retry_attempt}/{retry_max_attempt}] Snapshot ID: [{SNAPSHOT_CURRENT}/{sc.SNAPSHOT_TOTAL}]")
                 download_attempt = 1
                 download_max_attempt = 3
                 
@@ -247,19 +248,19 @@ def download_loop(output, worker, retry, no_redirect, delay):
                         if isinstance(e, (timeout, ConnectionRefusedError, ConnectionResetError)):
                             if download_attempt < download_max_attempt:
                                 download_attempt += 1  # try again 2x with same connection
-                                vb.write(message=f"\n-----> Worker: {worker} - Attempt: [{retry_attempt}/{retry_max_attempt}] Snapshot [{SNAPSHOT_CURRENT}/{sc.SNAPSHOT_TOTAL}] - {e.__class__.__name__} - requesting again in 50 seconds...")
+                                vb.write(message=f"\n-----> Worker: {worker} - Attempt: [{retry_attempt}/{retry_max_attempt}] Snapshot ID: [{SNAPSHOT_CURRENT}/{sc.SNAPSHOT_TOTAL}] - {e.__class__.__name__} - requesting again in 50 seconds...")
                                 time.sleep(50)
                                 continue
                         elif isinstance(e, http.client.HTTPException):
                             if download_attempt < download_max_attempt:
                                 download_attempt = download_max_attempt  # try again 1x with new connection
-                                vb.write(message=f"\n-----> Worker: {worker} - Attempt: [{retry_attempt}/{retry_max_attempt}] Snapshot [{SNAPSHOT_CURRENT}/{sc.SNAPSHOT_TOTAL}] - {e.__class__.__name__} - renewing connection in 15 seconds...")
+                                vb.write(message=f"\n-----> Worker: {worker} - Attempt: [{retry_attempt}/{retry_max_attempt}] Snapshot ID: [{SNAPSHOT_CURRENT}/{sc.SNAPSHOT_TOTAL}] - {e.__class__.__name__} - renewing connection in 15 seconds...")
                                 time.sleep(15)
                                 connection.close()
                                 connection = http.client.HTTPSConnection("web.archive.org")
                                 continue
                         else:
-                            ex.exception(message=f"\n-----> Worker: {worker} - Attempt: [{retry_attempt}/{retry_max_attempt}] Snapshot [{SNAPSHOT_CURRENT}/{sc.SNAPSHOT_TOTAL}] - EXCEPTION - {e}", e=e)
+                            ex.exception(message=f"\n-----> Worker: {worker} - Attempt: [{retry_attempt}/{retry_max_attempt}] Snapshot ID: [{SNAPSHOT_CURRENT}/{sc.SNAPSHOT_TOTAL}] - EXCEPTION - {e}", e=e)
                             retry_attempt = retry_max_attempt
                             break
 
@@ -272,11 +273,11 @@ def download_loop(output, worker, retry, no_redirect, delay):
                     
                     # depends on user - retries after timeout or proceed to next snapshot
                     if retry > 0:
-                        status_message.store(message=f"\n-----> Worker: {worker} - Attempt: [{retry_attempt}/{retry_max_attempt}] Snapshot [{SNAPSHOT_CURRENT}/{sc.SNAPSHOT_TOTAL}] - Download failed - retry Timeout: 15 seconds...")
+                        status_message.store(message=f"\n-----> Worker: {worker} - Attempt: [{retry_attempt}/{retry_max_attempt}] Snapshot ID: [{SNAPSHOT_CURRENT}/{sc.SNAPSHOT_TOTAL}] - Download failed - retry Timeout: 15 seconds...")
                         status_message.write()
                         time.sleep(15)
                     else:
-                        status_message.store(message=f"\n-----> Worker: {worker} - Attempt: [{retry_attempt}/{retry_max_attempt}] Snapshot [{SNAPSHOT_CURRENT}/{sc.SNAPSHOT_TOTAL}] - Download failed")
+                        status_message.store(message=f"\n-----> Worker: {worker} - Attempt: [{retry_attempt}/{retry_max_attempt}] Snapshot ID: [{SNAPSHOT_CURRENT}/{sc.SNAPSHOT_TOTAL}] - Download failed")
                         status_message.write()
                     sc.SNAPSHOT_HANDLED += 1
                     break # break all loops and do a user-defined retry

{pywaybackup-3.0.2 → pywaybackup-3.0.4}/pywaybackup/main.py

@@ -19,13 +19,13 @@ def main():
     db.init(config.dbfile, config.query_identifier)
     sc.init(config.mode)
 
-    archive.startup()
-
     if config.save:
         archive.save_page(config.url)
 
     else:
 
+        archive.startup()
+
         try:
             archive.query_list(config.csvfile, config.cdxfile, config.range, config.limit, config.start, config.end, config.explicit, config.filetype)
             archive.download_list(config.output, config.retry, config.no_redirect, config.delay, config.workers)

{pywaybackup-3.0.2 → pywaybackup-3.0.4/pywaybackup.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: pywaybackup
-Version: 3.0.2
+Version: 3.0.4
 Summary: Query and download archive.org as simple as possible.
 Author-email: bitdruid <bitdruid@outlook.com>
 License: MIT License