pywaybackup 3.0.3__tar.gz → 3.0.4__tar.gz

{pywaybackup-3.0.3/pywaybackup.egg-info → pywaybackup-3.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: pywaybackup
-Version: 3.0.3
+Version: 3.0.4
 Summary: Query and download archive.org as simple as possible.
 Author-email: bitdruid <bitdruid@outlook.com>
 License: MIT License

{pywaybackup-3.0.3 → pywaybackup-3.0.4}/pyproject.toml

@@ -7,7 +7,7 @@ packages = ["pywaybackup"]
 
 [project]
 name = "pywaybackup"
-version = "3.0.3"
+version = "3.0.4"
 description = "Query and download archive.org as simple as possible."
 authors = [
     { name = "bitdruid", email = "bitdruid@outlook.com" }

{pywaybackup-3.0.3 → pywaybackup-3.0.4}/pywaybackup/Arguments.py

@@ -74,13 +74,7 @@ class Configuration:
         
         if cls.output is None:
             cls.output = os.path.join(os.getcwd(), "waybackup_snapshots")
-
-        # check if output permissions are given
-        if not os.access(cls.output, os.W_OK):
-            print(f"\nNo write permissions for output folder: {cls.output}\n")
-            sys.exit(1)
-        else:
-            os.makedirs(cls.output, exist_ok=True)
+        os.makedirs(cls.output, exist_ok=True)
         
         if cls.log is True:
             cls.log = os.path.join(cls.output, f"waybackup_{sanitize_filename(cls.url)}.log")

{pywaybackup-3.0.3 → pywaybackup-3.0.4}/pywaybackup/SnapshotCollection.py

@@ -1,10 +1,15 @@
-from pywaybackup.Verbosity import Verbosity as vb
-from pywaybackup.helper import url_split
-from pywaybackup.db import Database
-from tqdm import tqdm
 import json
 import csv
 import os
+import threading
+
+from tqdm import tqdm
+
+from pywaybackup.Verbosity import Verbosity as vb
+from pywaybackup.helper import url_split
+from pywaybackup.db import Database
+
+LOCK = threading.Lock() # thread safe lock
 
 class SnapshotCollection:
     """
@@ -278,27 +283,31 @@ class SnapshotCollection:
         """
         Modify a snapshot-row in the snapshot table.
         """
-        connection.cursor.execute(
-            f"""
-            UPDATE snapshot_tbl
-            SET {column} = ?
-            WHERE rowid = ?
-            """,
-            (value, snapshot_id)
-        )
-        connection.conn.commit()
+        global LOCK
+        with LOCK:
+            connection.cursor.execute(
+                f"""
+                UPDATE snapshot_tbl
+                SET {column} = ?
+                WHERE rowid = ?
+                """,
+                (value, snapshot_id)
+            )
+            connection.conn.commit()
 
     def get_snapshot(connection):
         """
         Get a snapshot-row from the snapshot table with response NULL. (not processed)
         """
-        connection.cursor.execute(
-            """
-            SELECT rowid, * FROM snapshot_tbl WHERE response IS NULL LIMIT 1
-            """
-        )
-        row = connection.cursor.fetchone()
-        return row
+        global LOCK
+        with LOCK:
+            connection.cursor.execute(
+                """
+                SELECT rowid, * FROM snapshot_tbl WHERE response IS NULL LIMIT 1
+                """
+            )
+            row = connection.cursor.fetchone()
+            return row
 
     @classmethod
     def create_output(cls, url: str, timestamp: str, output: str):

{pywaybackup-3.0.3 → pywaybackup-3.0.4}/pywaybackup/archive.py

@@ -224,15 +224,16 @@ def download_loop(output, worker, retry, no_redirect, delay):
             
             snapshot = sc.get_snapshot(db)
             if not snapshot: break
-            sc.modify_snapshot(db, snapshot["rowid"], "response", "LOCK") # mark as locked for other workers
-            SNAPSHOT_CURRENT = sc.SNAPSHOT_HANDLED + 1
+            # mark as locked for other workers // only visual because get_snapshot fetches by NULL
+            sc.modify_snapshot(db, snapshot["rowid"], "response", "LOCK")
+            SNAPSHOT_CURRENT = snapshot["rowid"]
             
             retry_attempt = 1
             retry_max_attempt = retry if retry > 0 else retry + 1
             status_message = Message()
             
             while retry_attempt <= retry_max_attempt: # retry as given by user
-                status_message.store(message=f"\n-----> Worker: {worker} - Attempt: [{retry_attempt}/{retry_max_attempt}] Snapshot [{SNAPSHOT_CURRENT}/{sc.SNAPSHOT_TOTAL}]")
+                status_message.store(message=f"\n-----> Worker: {worker} - Attempt: [{retry_attempt}/{retry_max_attempt}] Snapshot ID: [{SNAPSHOT_CURRENT}/{sc.SNAPSHOT_TOTAL}]")
                 download_attempt = 1
                 download_max_attempt = 3
                 
@@ -247,19 +248,19 @@ def download_loop(output, worker, retry, no_redirect, delay):
                         if isinstance(e, (timeout, ConnectionRefusedError, ConnectionResetError)):
                             if download_attempt < download_max_attempt:
                                 download_attempt += 1  # try again 2x with same connection
-                                vb.write(message=f"\n-----> Worker: {worker} - Attempt: [{retry_attempt}/{retry_max_attempt}] Snapshot [{SNAPSHOT_CURRENT}/{sc.SNAPSHOT_TOTAL}] - {e.__class__.__name__} - requesting again in 50 seconds...")
+                                vb.write(message=f"\n-----> Worker: {worker} - Attempt: [{retry_attempt}/{retry_max_attempt}] Snapshot ID: [{SNAPSHOT_CURRENT}/{sc.SNAPSHOT_TOTAL}] - {e.__class__.__name__} - requesting again in 50 seconds...")
                                 time.sleep(50)
                                 continue
                         elif isinstance(e, http.client.HTTPException):
                             if download_attempt < download_max_attempt:
                                 download_attempt = download_max_attempt  # try again 1x with new connection
-                                vb.write(message=f"\n-----> Worker: {worker} - Attempt: [{retry_attempt}/{retry_max_attempt}] Snapshot [{SNAPSHOT_CURRENT}/{sc.SNAPSHOT_TOTAL}] - {e.__class__.__name__} - renewing connection in 15 seconds...")
+                                vb.write(message=f"\n-----> Worker: {worker} - Attempt: [{retry_attempt}/{retry_max_attempt}] Snapshot ID: [{SNAPSHOT_CURRENT}/{sc.SNAPSHOT_TOTAL}] - {e.__class__.__name__} - renewing connection in 15 seconds...")
                                 time.sleep(15)
                                 connection.close()
                                 connection = http.client.HTTPSConnection("web.archive.org")
                                 continue
                         else:
-                            ex.exception(message=f"\n-----> Worker: {worker} - Attempt: [{retry_attempt}/{retry_max_attempt}] Snapshot [{SNAPSHOT_CURRENT}/{sc.SNAPSHOT_TOTAL}] - EXCEPTION - {e}", e=e)
+                            ex.exception(message=f"\n-----> Worker: {worker} - Attempt: [{retry_attempt}/{retry_max_attempt}] Snapshot ID: [{SNAPSHOT_CURRENT}/{sc.SNAPSHOT_TOTAL}] - EXCEPTION - {e}", e=e)
                             retry_attempt = retry_max_attempt
                             break
 
@@ -272,11 +273,11 @@ def download_loop(output, worker, retry, no_redirect, delay):
                     
                     # depends on user - retries after timeout or proceed to next snapshot
                     if retry > 0:
-                        status_message.store(message=f"\n-----> Worker: {worker} - Attempt: [{retry_attempt}/{retry_max_attempt}] Snapshot [{SNAPSHOT_CURRENT}/{sc.SNAPSHOT_TOTAL}] - Download failed - retry Timeout: 15 seconds...")
+                        status_message.store(message=f"\n-----> Worker: {worker} - Attempt: [{retry_attempt}/{retry_max_attempt}] Snapshot ID: [{SNAPSHOT_CURRENT}/{sc.SNAPSHOT_TOTAL}] - Download failed - retry Timeout: 15 seconds...")
                         status_message.write()
                         time.sleep(15)
                     else:
-                        status_message.store(message=f"\n-----> Worker: {worker} - Attempt: [{retry_attempt}/{retry_max_attempt}] Snapshot [{SNAPSHOT_CURRENT}/{sc.SNAPSHOT_TOTAL}] - Download failed")
+                        status_message.store(message=f"\n-----> Worker: {worker} - Attempt: [{retry_attempt}/{retry_max_attempt}] Snapshot ID: [{SNAPSHOT_CURRENT}/{sc.SNAPSHOT_TOTAL}] - Download failed")
                         status_message.write()
                     sc.SNAPSHOT_HANDLED += 1
                     break # break all loops and do a user-defined retry

{pywaybackup-3.0.3 → pywaybackup-3.0.4/pywaybackup.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: pywaybackup
-Version: 3.0.3
+Version: 3.0.4
 Summary: Query and download archive.org as simple as possible.
 Author-email: bitdruid <bitdruid@outlook.com>
 License: MIT License