pywaybackup 0.2.7__tar.gz → 0.3.0__tar.gz

{pywaybackup-0.2.7 → pywaybackup-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pywaybackup
-Version: 0.2.7
+Version: 0.3.0
 Summary: Download snapshots from the Wayback Machine
 Home-page: https://github.com/bitdruid/waybackup
 Author: bitdruid
@@ -12,7 +12,7 @@ License-File: LICENSE
 
 # archive wayback downloader
 
-![Version](https://img.shields.io/badge/Version-0.2.7-blue)
+![Version](https://img.shields.io/badge/Version-0.3.0-blue)
 ![Release](https://img.shields.io/badge/Release-alpha-red)
 ![Python Version](https://img.shields.io/badge/Python-3.6-blue)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)

{pywaybackup-0.2.7 → pywaybackup-0.3.0}/README.md

@@ -1,6 +1,6 @@
 # archive wayback downloader
 
-![Version](https://img.shields.io/badge/Version-0.2.7-blue)
+![Version](https://img.shields.io/badge/Version-0.3.0-blue)
 ![Release](https://img.shields.io/badge/Release-alpha-red)
 ![Python Version](https://img.shields.io/badge/Python-3.6-blue)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)

pywaybackup-0.3.0/pywaybackup/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.3.0"

{pywaybackup-0.2.7 → pywaybackup-0.3.0}/pywaybackup/archive.py

@@ -4,6 +4,7 @@ import requests
 import datetime
 import os
 import magic
+import threading
 from pprint import pprint
 import time
 import pathlib
@@ -123,15 +124,12 @@ def remove_empty_folders(path, remove_root=True):
 # example download: http://web.archive.org/web/20190815104545id_/https://www.google.com/
 # example url: https://www.google.com/
 # example timestamp: 20190815104545
-def download_url_list(cdxResult_list, output, retry, mode):
+def download_prepare_list(cdxResult_list, output, retry, workers, mode):
     """
     Download a list of urls in format: [{"timestamp": "20190815104545", "url": "https://www.google.com/"}]
     """
-    #def download_batch(cdxResult_list):
     print("\nDownloading latest snapshots of each file...")
-    failed_urls = []
     download_list = []
-    connection = http.client.HTTPSConnection("web.archive.org")
     for snapshot in cdxResult_list:
         timestamp, url = snapshot["timestamp"], snapshot["url"]
         type = determine_url_filetype(url)
@@ -139,28 +137,32 @@ def download_url_list(cdxResult_list, output, retry, mode):
         domain, subdir, filename = split_url(url)
         if mode == "current": download_dir = os.path.join(output, domain, subdir)
         if mode == "full": download_dir = os.path.join(output, domain, timestamp, subdir)
-        download_filepath = os.path.join(download_dir, filename)
         download_list.append({"url": download_url, "filename": filename, "filepath": download_dir})
-    # download urls
-    for download_entry in download_list:
-        print(f"\n-----> Snapshot [{download_list.index(download_entry) + 1}/{len(download_list)}]")
-        download_url, download_filename, download_filepath = download_entry["url"], download_entry["filename"], download_entry["filepath"]
-        download_status=download_url_entry(download_url, download_filename, download_filepath, connection)
+    if workers > 1:
+        print(f"\n-----> Simultaneous downloads: {workers}")
+    threads = []
+    worker = 0
+    batch_size = len(download_list) // workers + 1
+    batch_list = [download_list[i:i + batch_size] for i in range(0, len(download_list), batch_size)]
+    for batch in batch_list:
+        worker += 1
+        thread = threading.Thread(target=download_url_list, args=(batch, worker, retry))
+        threads.append(thread)
+        thread.start()
+    for thread in threads:
+        thread.join()
+
+def download_url_list(url_list, worker, retry):
+    failed_urls = []
+    connection = http.client.HTTPSConnection("web.archive.org")
+    for url_entry in url_list:
+        status = f"\n-----> Snapshot [{url_list.index(url_entry) + 1}/{len(url_list)}] Worker: {worker}"
+        download_url, download_filename, download_filepath = url_entry["url"], url_entry["filename"], url_entry["filepath"]
+        download_status=download_url_entry(download_url, download_filename, download_filepath, connection, status)
         if download_status != bool(1): failed_urls.append({"url": download_url, "filename": download_filename, "filepath": download_filepath})
     if retry:
-        print(f"\n-----> Fail downloads: {len(failed_urls)}")
         download_retry(failed_urls, retry, connection)
     connection.close()
-    
-    # batch_size = len(download_list) // 10
-    # batch_list = [download_list[i:i + batch_size] for i in range(0, len(download_list), batch_size)]
-    # for batch in batch_list:
-    #     threads = []
-    #     thread = threading.Thread(target=download_batch, args=(batch,))
-    #     thread.start()
-    #     threads.append(thread)
-    # for thread in threads:
-    #     thread.join()
 
 def download_retry(failed_urls, retry, connection):
     """
@@ -174,16 +176,16 @@ def download_retry(failed_urls, retry, connection):
         print(f"\n-----> Retrying...")
         retry_urls = []
         for failed_entry in failed_urls:
+            status = f"\n-----> RETRY attempt: [{attempt}/{max_attempt}] Snapshot [{failed_urls.index(failed_entry) + 1}/{len(failed_urls)}]"
             download_url, download_filename, download_filepath = failed_entry["url"], failed_entry["filename"], failed_entry["filepath"]
-            print(f"\n-----> RETRY attempt: [{attempt}/{max_attempt}] Snapshot [{failed_urls.index(failed_entry) + 1}/{len(failed_urls)}]")
-            retry_status=download_url_entry(download_url, download_filename, download_filepath, connection)
+            retry_status=download_url_entry(download_url, download_filename, download_filepath, connection, status)
             if retry_status != bool(1):
                 retry_urls.append({"url": download_url, "filename": download_filename, "filepath": download_filepath})
         failed_urls = retry_urls
         print(f"\n-----> Fail downloads: {len(failed_urls)}")
         if retry != None: attempt += 1
 
-def download_url_entry(url, filename, filepath, connection):
+def download_url_entry(url, filename, filepath, connection, status):
     """
     Download a single url.
     Success: return bool(1)
@@ -201,10 +203,12 @@ def download_url_entry(url, filename, filepath, connection):
             data = response.read()
             with open(output, 'wb') as file:
                 file.write(data)
+            print(status)
             print(f"SUCCESS -> {url}")
             print(f"        -> {output}")
             return bool(1)
         except http.client.HTTPException as e:
+            print(status)
             print(f"REFUSED -> ({i+1}/{max_retries}), reconnect in {sleep_time} seconds...")
             time.sleep(sleep_time)
     print(f"FAILED  -> download, append to failed_urls: {url}")

{pywaybackup-0.2.7 → pywaybackup-0.3.0}/pywaybackup/main.py

@@ -19,6 +19,7 @@ def main():
     special = parser.add_argument_group('special')
     special.add_argument('--detect-filetype', action='store_true', help='If a file has no extension, try to detect the filetype')
     special.add_argument('--retry-failed', nargs='?', const=True, type=int, help='Retry failed downloads (opt tries as int, else infinite)')
+    special.add_argument('--workers', type=int, default=1, help='Number of workers (simultaneous downloads)')
 
     args = parser.parse_args()
     if args.current:
@@ -32,7 +33,7 @@ def main():
     if args.list:
         archive.print_result(cdxResult_list)
     if not args.list:
-        archive.download_url_list(cdxResult_list, args.output, args.retry_failed, mode)
+        archive.download_prepare_list(cdxResult_list, args.output, args.retry_failed, args.workers, mode)
         archive.remove_empty_folders(args.output)
     if args.detect_filetype:
         archive.detect_filetype(args.output)

{pywaybackup-0.2.7 → pywaybackup-0.3.0}/pywaybackup.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pywaybackup
-Version: 0.2.7
+Version: 0.3.0
 Summary: Download snapshots from the Wayback Machine
 Home-page: https://github.com/bitdruid/waybackup
 Author: bitdruid
@@ -12,7 +12,7 @@ License-File: LICENSE
 
 # archive wayback downloader
 
-![Version](https://img.shields.io/badge/Version-0.2.7-blue)
+![Version](https://img.shields.io/badge/Version-0.3.0-blue)
 ![Release](https://img.shields.io/badge/Release-alpha-red)
 ![Python Version](https://img.shields.io/badge/Python-3.6-blue)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)

pywaybackup-0.2.7/pywaybackup/__version__.py

@@ -1 +0,0 @@
-__version__ = "0.2.7"