pywaybackup 0.6.3__tar.gz → 0.7.1__tar.gz

{pywaybackup-0.6.3/pywaybackup.egg-info → pywaybackup-0.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pywaybackup
-Version: 0.6.3
+Version: 0.7.1
 Summary: Download snapshots from the Wayback Machine
 Home-page: https://github.com/bitdruid/python-wayback-machine-downloader
 Author: bitdruid
@@ -14,7 +14,7 @@ License-File: LICENSE
 
 [![PyPI](https://img.shields.io/pypi/v/pywaybackup)](https://pypi.org/project/pywaybackup/)
 [![PyPI - Downloads](https://img.shields.io/pypi/dm/pywaybackup)](https://pypi.org/project/pywaybackup/)
-![Release](https://img.shields.io/badge/Release-alpha-red)
+![Release](https://img.shields.io/badge/Release-beta-orange)
 ![Python Version](https://img.shields.io/badge/Python-3.6-blue)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 
@@ -43,6 +43,8 @@ Internet-archive is a nice source for several OSINT-information. This script is
 
 This script allows you to download content from the Wayback Machine (archive.org). You can use it to download either the latest version or all versions of web page snapshots within a specified range.
 
+<!-- ## Info -->
+
 ### Arguments
 
 - `-h`, `--help`: Show the help message and exit.
@@ -54,8 +56,8 @@ This script allows you to download content from the Wayback Machine (archive.org
 
 #### Mode Selection (Choose One)
 
-- `-c`, `--current`: Download the latest version of each file snapshot.
-- `-f`, `--full`: Download snapshots of all timestamps.
+- `-c`, `--current`: Download the latest version of each file snapshot. You will get a rebuild of the current website with all available files.
+- `-f`, `--full`: Download snapshots of all timestamps. You will get a folder per timestamp with the files available at that time.
 - `-s`, `--save`: Save a page to the Wayback Machine. (beta)
 
 #### Optional Arguments
@@ -73,6 +75,7 @@ Specify the range in years or a specific timestamp either start, end or both. If
 
 #### Additional
 
+- `--no-redirect`: Do not follow redirects of snapshots. Archive.org sometimes redirects to a different snapshot for several reasons. Downloading redirects may lead to timestamp-folders which contain some files with a different timestamp. This does not matter if you only want to download the latest version (`-c`).
 - `--verbosity [LEVEL]`: Set the verbosity: json (print json response), progress (show progress bar) or standard (default).
 - `--retry [RETRY_FAILED]`: Retry failed downloads. You can specify the number of retry attempts as an integer.
 - `--worker [AMOUNT]`: The number of worker to use for downloading (simultaneous downloads). Default is 1. Beware: Using too many worker will lead into refused connections from the Wayback Machine. Duration about 1.5 minutes.
@@ -85,8 +88,8 @@ Download latest snapshot of all files:<br>
 Download latest snapshot of all files with retries:<br>
 `waybackup -u http://example.com -c --retry 3`
 
-Download all snapshots sorted per timestamp with a specified range:<br>
-`waybackup -u http://example.com -f -r 5`
+Download all snapshots sorted per timestamp with a specified range and follow redirects:<br>
+`waybackup -u http://example.com -f -r 5 --redirect`
 
 Download all snapshots sorted per timestamp with a specified range and save to a specified folder with 3 worker:<br>
 `waybackup -u http://example.com -f -r 5 -o /home/user/Downloads/snapshots --worker 3`
@@ -94,10 +97,6 @@ Download all snapshots sorted per timestamp with a specified range and save to a
 List available snapshots per timestamp without downloading:<br>
 `waybackup -u http://example.com -f -l`
 
-## Info
-
-The script automatically follows redirects of snapshots.
-
 ## Contributing
 
 I'm always happy for some feature requests to improve the usability of this script.

{pywaybackup-0.6.3 → pywaybackup-0.7.1}/README.md

@@ -2,7 +2,7 @@
 
 [![PyPI](https://img.shields.io/pypi/v/pywaybackup)](https://pypi.org/project/pywaybackup/)
 [![PyPI - Downloads](https://img.shields.io/pypi/dm/pywaybackup)](https://pypi.org/project/pywaybackup/)
-![Release](https://img.shields.io/badge/Release-alpha-red)
+![Release](https://img.shields.io/badge/Release-beta-orange)
 ![Python Version](https://img.shields.io/badge/Python-3.6-blue)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 
@@ -31,6 +31,8 @@ Internet-archive is a nice source for several OSINT-information. This script is
 
 This script allows you to download content from the Wayback Machine (archive.org). You can use it to download either the latest version or all versions of web page snapshots within a specified range.
 
+<!-- ## Info -->
+
 ### Arguments
 
 - `-h`, `--help`: Show the help message and exit.
@@ -42,8 +44,8 @@ This script allows you to download content from the Wayback Machine (archive.org
 
 #### Mode Selection (Choose One)
 
-- `-c`, `--current`: Download the latest version of each file snapshot.
-- `-f`, `--full`: Download snapshots of all timestamps.
+- `-c`, `--current`: Download the latest version of each file snapshot. You will get a rebuild of the current website with all available files.
+- `-f`, `--full`: Download snapshots of all timestamps. You will get a folder per timestamp with the files available at that time.
 - `-s`, `--save`: Save a page to the Wayback Machine. (beta)
 
 #### Optional Arguments
@@ -61,6 +63,7 @@ Specify the range in years or a specific timestamp either start, end or both. If
 
 #### Additional
 
+- `--no-redirect`: Do not follow redirects of snapshots. Archive.org sometimes redirects to a different snapshot for several reasons. Downloading redirects may lead to timestamp-folders which contain some files with a different timestamp. This does not matter if you only want to download the latest version (`-c`).
 - `--verbosity [LEVEL]`: Set the verbosity: json (print json response), progress (show progress bar) or standard (default).
 - `--retry [RETRY_FAILED]`: Retry failed downloads. You can specify the number of retry attempts as an integer.
 - `--worker [AMOUNT]`: The number of worker to use for downloading (simultaneous downloads). Default is 1. Beware: Using too many worker will lead into refused connections from the Wayback Machine. Duration about 1.5 minutes.
@@ -73,8 +76,8 @@ Download latest snapshot of all files:<br>
 Download latest snapshot of all files with retries:<br>
 `waybackup -u http://example.com -c --retry 3`
 
-Download all snapshots sorted per timestamp with a specified range:<br>
-`waybackup -u http://example.com -f -r 5`
+Download all snapshots sorted per timestamp with a specified range and follow redirects:<br>
+`waybackup -u http://example.com -f -r 5 --redirect`
 
 Download all snapshots sorted per timestamp with a specified range and save to a specified folder with 3 worker:<br>
 `waybackup -u http://example.com -f -r 5 -o /home/user/Downloads/snapshots --worker 3`
@@ -82,10 +85,6 @@ Download all snapshots sorted per timestamp with a specified range and save to a
 List available snapshots per timestamp without downloading:<br>
 `waybackup -u http://example.com -f -l`
 
-## Info
-
-The script automatically follows redirects of snapshots.
-
 ## Contributing
 
 I'm always happy for some feature requests to improve the usability of this script.

pywaybackup-0.7.1/pywaybackup/SnapshotCollection.py

@@ -0,0 +1,110 @@
+from urllib.parse import urlparse
+import os
+
+class SnapshotCollection:
+
+    CDX_LIST = []
+    SNAPSHOT_COLLECTION = []
+    MODE_CURRENT = 0
+
+    @classmethod
+    def create_list_full(cls, cdxResult):
+        cls.CDX_LIST = sorted([{"timestamp": snapshot[0], "url": snapshot[1], "status": snapshot[2], "mimetype": snapshot[3], "digest": snapshot[4]} for i, snapshot in enumerate(cdxResult.json()[1:])], key=lambda k: k['timestamp'], reverse=True)
+
+    @classmethod
+    def create_list_current(cls):
+        cls.MODE_CURRENT = 1
+        cdxResult_list_filtered = []
+        url_set = set()
+        for snapshot in cls.CDX_LIST:
+            if snapshot["url"] not in url_set:
+                cdxResult_list_filtered.append(snapshot)
+                url_set.add(snapshot["url"])
+        cls.CDX_LIST = cdxResult_list_filtered
+
+    @classmethod
+    def count_list(cls):
+        return len(cls.CDX_LIST)
+
+    @classmethod
+    def create_collection(cls):
+        for cdx_entry in cls.CDX_LIST:
+            timestamp, url = cdx_entry["timestamp"], cdx_entry["url"]
+            url_archive = f"http://web.archive.org/web/{timestamp}{cls._url_get_filetype(url)}/{url}"
+            collection_entry = {
+                "id": len(cls.SNAPSHOT_COLLECTION),
+                "timestamp": timestamp,
+                "url_archive": url_archive,
+                "url_origin": url,
+                "file": False,
+                "redirect": False,
+                "response": False
+            }
+            cls.SNAPSHOT_COLLECTION.append(collection_entry)
+    
+    @classmethod
+    def snapshot_entry_create_output(cls, collection_entry: dict, output: str) -> str:
+        """
+        Create the output path for a snapshot entry of the collection according to the mode.
+
+        Input:
+        - collection_entry: A single snapshot entry of the collection (dict).
+        - output: The output directory (str).
+
+        Output:
+        - download_file: The output path for the snapshot entry (str) with filename.
+        """
+        timestamp, url = collection_entry["timestamp"], collection_entry["url_origin"]
+        domain, subdir, filename = cls._url_split(url)
+        if cls.MODE_CURRENT:
+            download_dir = os.path.join(output, domain, subdir)
+        else:
+            download_dir = os.path.join(output, domain, timestamp, subdir)
+        download_file = os.path.join(download_dir, filename)
+        return download_file
+
+    @classmethod
+    def snapshot_entry_modify(cls, collection_entry: dict, key: str, value: str):
+        """
+        Modify a key-value pair in a snapshot entry of the collection (dict).
+
+        - Append a new key-value pair if the key does not exist.
+        - Modify an existing key-value pair if the key exists.
+        """
+        collection_entry[key] = value
+
+    @classmethod
+    def url_get_timestamp(cls, url):
+        """
+        Extract the timestamp from a wayback machine URL.
+        """
+        timestamp = url.split("web.archive.org/web/")[1].split("/")[0]
+        timestamp = ''.join([char for char in timestamp if char.isdigit()])
+        return timestamp
+
+    @classmethod
+    def _url_get_filetype(cls, url):
+        file_extension = os.path.splitext(url)[1][1:]
+        urltype_mapping = {
+            "jpg": "im_",
+            "jpeg": "im_",
+            "png": "im_",
+            "gif": "im_",
+            "svg": "im_",
+            "ico": "im_",
+            "css": "cs_"
+            #"js": "js_"
+        }
+        urltype = urltype_mapping.get(file_extension, "id_")
+        return urltype
+
+    @classmethod
+    def _url_split(cls, url):
+        """
+        Split a URL into domain, subdir and filename.
+        """
+        parsed_url = urlparse(url)
+        domain = parsed_url.netloc
+        subdir = parsed_url.path.strip("/").rsplit("/", 1)[0]
+        filename = parsed_url.path.split("/")[-1] or "index.html"
+        return domain, subdir, filename

{pywaybackup-0.6.3 → pywaybackup-0.7.1}/pywaybackup/Verbosity.py

@@ -1,7 +1,6 @@
 import tqdm
 import json
-import time
-import pywaybackup.SnapshotCollection as sc
+from pywaybackup.SnapshotCollection import SnapshotCollection as sc
 
 class Verbosity:
 
@@ -11,8 +10,7 @@ class Verbosity:
     pbar = None
 
     @classmethod
-    def open(cls, args: list, snapshots: sc.SnapshotCollection):
-        cls.snapshots = snapshots
+    def open(cls, args: list):
         cls.args = args
         if cls.args == "progress":
             cls.mode = "progress"
@@ -23,16 +21,23 @@ class Verbosity:
 
     @classmethod
     def close(cls):
-        if cls.mode == "json":
-            print(json.dumps(cls.snapshots.SNAPSHOT_COLLECTION, indent=4, sort_keys=True))
-        elif cls.mode == "standard":
+        if cls.mode == "progress":
+            cls.pbar.close()
+        if cls.mode == "progress" or cls.mode == "standard":
+            successed = len([snapshot for snapshot in sc.SNAPSHOT_COLLECTION if snapshot["file"]])
+            failed = len([snapshot for snapshot in sc.SNAPSHOT_COLLECTION if not snapshot["file"]])
+            print(f"\nFiles downloaded: {successed}")
+            print(f"Files missing: {failed}")
             print("")
+        if cls.mode == "json":
+            print(json.dumps(sc.SNAPSHOT_COLLECTION, indent=4, sort_keys=True))
 
     @classmethod
     def write(cls, message: str = None, progress: int = None):
         if cls.mode == "progress":
             if progress == 0:
-                maxval = cls.snapshots.count_list()
+                print("")
+                maxval = sc.count_list()
                 cls.pbar = tqdm.tqdm(total=maxval, desc="Downloading", unit=" snapshot", ascii="░▒█")
             elif progress == 1:
                 cls.pbar.update(1)

pywaybackup-0.7.1/pywaybackup/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.7.1"

{pywaybackup-0.6.3 → pywaybackup-0.7.1}/pywaybackup/archive.py

@@ -1,4 +1,3 @@
-#import threading
 import requests
 import os
 import gzip
@@ -8,7 +7,8 @@ import http.client
 from urllib.parse import urljoin
 from datetime import datetime, timezone
 
-import pywaybackup.SnapshotCollection as sc
+from pywaybackup.SnapshotCollection import SnapshotCollection as sc
+
 from pywaybackup.Verbosity import Verbosity as v
 
 
@@ -42,7 +42,7 @@ def save_page(url: str):
     if response_status == 302:
         location = response.getheader("Location")
         v.write("\n-----> Response: 302 (redirect to snapshot)")
-        snapshot_timestamp = datetime.strptime(location.split('/web/')[1].split('/')[0], '%Y%m%d%H%M%S').strftime('%Y-%m-%d %H:%M:%S')
+        snapshot_timestamp = datetime.strptime(sc.url_get_timestamp(location), '%Y%m%d%H%M%S').strftime('%Y-%m-%d %H:%M:%S')
         current_timestamp = datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S')
         timestamp_difference = (datetime.strptime(current_timestamp, '%Y-%m-%d %H:%M:%S') - datetime.strptime(snapshot_timestamp, '%Y-%m-%d %H:%M:%S')).seconds / 60
         timestamp_difference = int(round(timestamp_difference, 0))
@@ -70,13 +70,14 @@ def save_page(url: str):
 
 
 
-def print_list(snapshots):
+def print_list():
     v.write("")
-    if not snapshots:
-        v.write("No snapshots found")
+    count = sc.count_list()
+    if count == 0:
+        v.write("\nNo snapshots found")
     else:
-        __import__('pprint').pprint(snapshots.CDX_LIST)
-        v.write(f"\n-----> {snapshots.count_list()} snapshots listed")
+        __import__('pprint').pprint(sc.CDX_LIST)
+        v.write(f"\n-----> {count} snapshots listed")
 
 
 
@@ -84,20 +85,21 @@ def print_list(snapshots):
 
 # create filelist
 # timestamp format yyyyMMddhhmmss
-def query_list(snapshots: sc.SnapshotCollection, url: str, range: int, start: int, end: int, explicit: bool, mode: str):
+def query_list(url: str, range: int, start: int, end: int, explicit: bool, mode: str):
     try:
         v.write("\nQuerying snapshots...")
-        range = ""
+        query_range = ""
         if not range:
-            if start: range = range + f"&from={start}"
-            if end: range = range + f"&to={end}"
-        else: range = "&from=" + str(datetime.now().year - range)
+            if start: query_range = query_range + f"&from={start}"
+            if end: query_range = query_range + f"&to={end}"
+        else: 
+            query_range = "&from=" + str(datetime.now().year - range)
         cdx_url = f"*.{url}/*" if not explicit else f"{url}"
-        cdxQuery = f"https://web.archive.org/cdx/search/xd?output=json&url={cdx_url}{range}&fl=timestamp,original&filter=!statuscode:200"
+        cdxQuery = f"https://web.archive.org/cdx/search/xd?output=json&url={cdx_url}{query_range}&fl=timestamp,original,statuscode,mimetype,digest&filter!=statuscode:200"
         cdxResult = requests.get(cdxQuery)
-        snapshots.create_full(cdxResult)
-        if mode == "current": snapshots.create_current()
-        v.write(f"\n-----> {snapshots.count_list()} snapshots found")
+        sc.create_list_full(cdxResult)
+        sc.create_list_current() if mode == "current" else None
+        v.write(f"\n-----> {sc.count_list()} snapshots found")
     except requests.exceptions.ConnectionError as e:
         v.write(f"\n-----> ERROR: could not query snapshots:\n{e}"); exit()
 
@@ -105,108 +107,68 @@ def query_list(snapshots: sc.SnapshotCollection, url: str, range: int, start: in
 
 
 
-
-def remove_empty_folders(path, remove_root=True):
-    v.write("")
-    v.write("Removing empty output folders...")
-    count = 0
-    if not os.path.isdir(path):
-        return
-    # remove empty subfolders
-    for root, dirs, files in os.walk(path, topdown=False):
-        for dir in dirs:
-            dir_path = os.path.join(root, dir)
-            if not os.listdir(dir_path):
-                try:
-                    os.rmdir(dir_path)
-                    v.write(f"-----> {dir_path}")
-                    count += 1
-                except OSError as e:
-                    v.write(f"Error removing {dir_path}: {e}")
-    # remove empty root folder
-    if remove_root and not os.listdir(path):
-        try:
-            os.rmdir(path)
-            v.write(f"-----> {path}")
-            count += 1
-        except OSError as e:
-            v.write(f"Error removing {path}: {e}")
-    if count == 0:
-        v.write("No empty folders found")
-
-
-
-
-
 # example download: http://web.archive.org/web/20190815104545id_/https://www.google.com/
-def download_list(snapshots, output, retry, worker):
+def download_list(output, retry, no_redirect, worker):
     """
     Download a list of urls in format: [{"timestamp": "20190815104545", "url": "https://www.google.com/"}]
     """
-    if snapshots.count_list() == 0: 
-        v.write("\nNo snapshots found to download")
+    if sc.count_list() == 0: 
+        v.write("\nNothing to download");
         return
-    v.write("\nDownloading latest snapshots of each file...", progress=0)
-    download_list = snapshots.CDX_LIST
+    v.write("\nDownloading snapshots...", progress=0)
     if worker > 1:
         v.write(f"\n-----> Simultaneous downloads: {worker}")
-        batch_size = snapshots.count_list() // worker + 1
+        batch_size = sc.count_list() // worker + 1
     else:
-        batch_size = snapshots.count_list()
-    batch_list = [download_list[i:i + batch_size] for i in range(0, len(download_list), batch_size)]
+        batch_size = sc.count_list()
+    sc.create_collection()
+    batch_list = [sc.SNAPSHOT_COLLECTION[i:i + batch_size] for i in range(0, len(sc.SNAPSHOT_COLLECTION), batch_size)]    
     threads = []
     worker = 0
     for batch in batch_list:
         worker += 1
-        thread = threading.Thread(target=download_loop, args=(snapshots, batch, output, worker, retry))
+        thread = threading.Thread(target=download_loop, args=(batch, output, worker, retry, no_redirect))
         threads.append(thread)
         thread.start()
     for thread in threads:
         thread.join()
-    failed_urls = len([url for url in snapshots.SNAPSHOT_COLLECTION if not url["success"]])
-    if failed_urls: v.write(f"\n-----> Failed downloads: {failed_urls}")
 
-def download_loop(snapshots, cdx_list, output, worker, retry, attempt=1, connection=None):
+def download_loop(snapshot_batch, output, worker, retry, no_redirect, attempt=1, connection=None):
     """
     Download a list of URLs in a recursive loop. If a download fails, the function will retry the download.
     The "snapshot_collection" dictionary will be updated with the download status and file information.
-    Information for each entry is written by "create_entry" and "snapshot_collection_write" functions.
+    Information for each entry is written by "create_entry" and "snapshot_dict_append" functions.
     """
-    max_attempt = retry + 1
+    max_attempt = retry if retry > 0 else retry + 1
     failed_urls = []
     if not connection:
         connection = http.client.HTTPSConnection("web.archive.org")
-    if attempt > max_attempt: 
+    if attempt > max_attempt:
         connection.close()
-        v.write(f"\n-----> Worker: {worker} - Failed downloads: {len(cdx_list)}")
+        v.write(f"\n-----> Worker: {worker} - Failed downloads: {len(snapshot_batch)}")
         return
-    else:
-        for cdx_entry in cdx_list:
-            status = f"\n-----> Attempt: [{attempt}/{max_attempt}] Snapshot [{cdx_list.index(cdx_entry)+1}/{len(cdx_list)}] - Worker: {worker}"
-            download_entry = snapshots.create_entry(cdx_entry, output)
-            snapshots.snapshot_collection_write(download_entry)
-            download_status=download(download_entry, connection, status)
-            if not download_status:
-                snapshots.snapshot_collection_update(download_entry["id"], "success", False)
-                snapshots.snapshot_collection_update(download_entry["id"], "file", None)
-                snapshots.snapshot_collection_update(download_entry["id"], "retry", attempt)
-                failed_urls.append(cdx_entry);
-            if download_status:
-                snapshots.snapshot_collection_update(download_entry["id"], "success", True)
-                snapshots.snapshot_collection_update(download_entry["id"], "file", download_entry["file"])
-                v.write(progress=1)
-        attempt += 1
-    if failed_urls: download_loop(snapshots, failed_urls, output, worker, retry, attempt, connection)
-
-def download(download_entry, connection, status_message):
+    for snapshot in snapshot_batch:
+        status = f"\n-----> Attempt: [{attempt}/{max_attempt}] Snapshot [{snapshot_batch.index(snapshot)+1}/{len(snapshot_batch)}] - Worker: {worker}"
+        download_status = download(output, snapshot, connection, status, no_redirect)
+        if not download_status:
+            failed_urls.append(snapshot)
+        if download_status:
+            v.write(progress=1)
+    attempt += 1
+    if failed_urls:
+        if not attempt > max_attempt: 
+            v.write(f"\n-----> Worker: {worker} - Retry Timeout: 10 seconds")
+            time.sleep(15)
+        download_loop(failed_urls, output, worker, retry, no_redirect, attempt, connection)
+
+def download(output, snapshot_entry, connection, status_message, no_redirect=False):
     """
     Download a single URL and save it to the specified filepath.
     If there is a redirect, the function will follow the redirect and update the download URL.
     gzip decompression is used if the response is encoded.
     According to the response status, the function will write a status message to the console and append a failed URL.
     """
-    download_url = download_entry["url"]
-    download_file = download_entry["file"]
+    download_url = snapshot_entry["url_archive"]
     max_retries = 2
     sleep_time = 45
     headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36'}
@@ -216,23 +178,31 @@ def download(download_entry, connection, status_message):
             response = connection.getresponse()
             response_data = response.read()
             response_status = response.status
-            if response_status == 302:
-                status_message = f"{status_message}\n" + \
-                    f"REDIRECT   -> HTTP: {response.status}"
-                while response_status == 302:
-                    connection.request("GET", download_url, headers=headers)
-                    response = connection.getresponse()
-                    response_data = response.read()
-                    response_status = response.status
-                    location = response.getheader("Location")
-                    if location:
-                        status_message = f"{status_message}\n" + \
-                            f"           -> URL: {location}"
-                        location = urljoin(download_url, location)
-                        download_url = location
-                    else:
-                        break
-            if response_status != 404:
+            response_status_message = parse_response_code(response_status)
+            sc.snapshot_entry_modify(snapshot_entry, "response", response_status)
+            if not no_redirect:
+                if response_status == 302:
+                    status_message = f"{status_message}\n" + \
+                        f"REDIRECT   -> HTTP: {response.status}"
+                    while response_status == 302:
+                        connection.request("GET", download_url, headers=headers)
+                        response = connection.getresponse()
+                        response_data = response.read()
+                        response_status = response.status
+                        location = response.getheader("Location")
+                        if location:
+                            status_message = f"{status_message}\n" + \
+                                f"           -> URL: {location}"
+                            location = urljoin(download_url, location)
+                            download_url = location
+                            sc.snapshot_entry_modify(snapshot_entry, "redirect", True)
+                            sc.snapshot_entry_modify(snapshot_entry, "redirect_timestamp", sc.url_get_timestamp(location))
+                            sc.snapshot_entry_modify(snapshot_entry, "redirect_url", location)
+                        else:
+                            break
+            if response_status == 200:
+                sc.snapshot_entry_modify(snapshot_entry, "file", sc.snapshot_entry_create_output(snapshot_entry, output))
+                download_file = snapshot_entry["file"]
                 os.makedirs(os.path.dirname(download_file), exist_ok=True)
                 with open(download_file, 'wb') as file:
                     if response.getheader('Content-Encoding') == 'gzip':
@@ -240,33 +210,50 @@ def download(download_entry, connection, status_message):
                         file.write(response_data)
                     else:
                         file.write(response_data)
-            if response_status == 200:
-                status_message = f"{status_message}\n" + \
-                    f"SUCCESS    -> HTTP: {response.status}\n" + \
-                    f"           -> URL: {download_url}\n" + \
-                    f"           -> FILE: {download_file}"
-            elif response_status == 404:
-                status_message = f"{status_message}\n" + \
-                    f"NOT FOUND  -> HTTP: {response.status}\n" + \
-                    f"           -> URL: {download_url}"
+                if os.path.isfile(download_file):
+                    status_message = f"{status_message}\n" + \
+                        f"SUCCESS    -> HTTP: {response_status} - {response_status_message}\n" + \
+                        f"           -> URL: {download_url}\n" + \
+                        f"           -> FILE: {download_file}"
+                v.write(status_message)
+                return True
             else:
                 status_message = f"{status_message}\n" + \
-                    f"UNEXPECTED -> HTTP: {response.status}\n" + \
-                    f"           -> URL: {download_url}\n" + \
-                    f"           -> FILE: {download_file}"
+                    f"UNEXPECTED -> HTTP: {response_status} - {response_status_message}\n" + \
+                    f"           -> URL: {download_url}\n"
+                v.write(status_message)
+                return True
+        # exception returns false and appends the url to the failed list
+        except http.client.HTTPException as e:
+            status_message = f"{status_message}\n" + \
+                f"EXCEPTION -> ({i+1}/{max_retries}), append to failed_urls: {download_url}\n" + \
+                f"          -> {e}"
             v.write(status_message)
-            return True
+            return False
+        # connection refused waits and retries
         except ConnectionRefusedError as e:
             status_message = f"{status_message}\n" + \
                 f"REFUSED  -> ({i+1}/{max_retries}), reconnect in {sleep_time} seconds...\n" + \
                 f"         -> {e}"
             v.write(status_message)
             time.sleep(sleep_time)
-        except http.client.HTTPException as e:
-            status_message = f"{status_message}\n" + \
-                f"EXCEPTION -> ({i+1}/{max_retries}), append to failed_urls: {download_url}\n" + \
-                f"          -> {e}"
-            v.write(status_message)
-            return False
     v.write(f"FAILED  -> download, append to failed_urls: {download_url}")
-    return False
+    return False
+
+RESPONSE_CODE_DICT = {
+    200: "OK",
+    301: "Moved Permanently",
+    302: "Found (redirect)",
+    400: "Bad Request",
+    403: "Forbidden",
+    404: "Not Found",
+    500: "Internal Server Error",
+    503: "Service Unavailable"
+}
+def parse_response_code(response_code: int):
+    """
+    Parse the response code of the Wayback Machine and return a human-readable message.
+    """
+    if response_code in RESPONSE_CODE_DICT:
+        return RESPONSE_CODE_DICT[response_code]
+    return "Unknown response code"

pywaybackup-0.6.3/pywaybackup/main.py → pywaybackup-0.7.1/pywaybackup/arguments.py

@@ -1,21 +1,18 @@
-import pywaybackup.archive as archive
-import pywaybackup.SnapshotCollection as sc
 import argparse
-import os
-
-from pywaybackup.Verbosity import Verbosity as v
-
 from pywaybackup.__version__ import __version__
 
-def main():
+def parse():
+
     parser = argparse.ArgumentParser(description='Download from wayback machine (archive.org)')
     parser.add_argument('-a', '--about', action='version', version='%(prog)s ' + __version__ + ' by @bitdruid -> https://github.com/bitdruid')
+
     required = parser.add_argument_group('required')
     required.add_argument('-u', '--url', type=str, help='URL to use')
     exclusive_required = required.add_mutually_exclusive_group(required=True)
     exclusive_required.add_argument('-c', '--current', action='store_true', help='Download the latest version of each file snapshot (opt range in y)')
     exclusive_required.add_argument('-f', '--full', action='store_true', help='Download snapshots of all timestamps (opt range in y)')
     exclusive_required.add_argument('-s', '--save', action='store_true', help='Save a page to the wayback machine')
+
     optional = parser.add_argument_group('optional')
     optional.add_argument('-l', '--list', action='store_true', help='Only print snapshots (opt range in y)')
     optional.add_argument('-e', '--explicit', action='store_true', help='Search only for the explicit given url')
@@ -23,32 +20,13 @@ def main():
     optional.add_argument('-r', '--range', type=int, help='Range in years to search')
     optional.add_argument('--start', type=int, help='Start timestamp format: YYYYMMDDhhmmss')
     optional.add_argument('--end', type=int, help='End timestamp format: YYYYMMDDhhmmss')
+
     special = parser.add_argument_group('special')
+    special.add_argument('--no-redirect', action='store_true', help='Do not follow redirects by archive.org')
     special.add_argument('--verbosity', type=str, default="standard", choices=["standard", "progress", "json"], help='Verbosity level')
     special.add_argument('--retry', type=int, default=0, metavar="X-TIMES", help='Retry failed downloads (opt tries as int, else infinite)')
     special.add_argument('--worker', type=int, default=1, metavar="AMOUNT", help='Number of worker (simultaneous downloads)')
 
     args = parser.parse_args()
-    snapshots = sc.SnapshotCollection()
-    v.open(args.verbosity, snapshots)
-
-    if args.current:
-        mode = "current"
-    elif args.full:
-        mode = "full"
-
-    if args.save:
-        archive.save_page(args.url)
-    else:
-        if args.output is None:
-            args.output = os.path.join(os.getcwd(), "waybackup_snapshots")
-        archive.query_list(snapshots, args.url, args.range, args.start, args.end, args.explicit, mode)
-        if args.list:
-            archive.print_list(snapshots)
-        else:
-            archive.download_list(snapshots, args.output, args.retry, args.worker)            
-            archive.remove_empty_folders(args.output)
-    v.close()
 
-if __name__ == "__main__":
-    main()
+    return args

pywaybackup-0.7.1/pywaybackup/main.py

@@ -0,0 +1,31 @@
+import pywaybackup.archive as archive
+import pywaybackup.SnapshotCollection as sc
+import os
+
+from pywaybackup.arguments import parse
+from pywaybackup.Verbosity import Verbosity as v
+
+def main():
+    args = parse()
+    v.open(args.verbosity)
+
+    if args.current:
+        mode = "current"
+    elif args.full:
+        mode = "full"
+
+    if args.save:
+        archive.save_page(args.url)
+    else:
+        if args.output is None:
+            args.output = os.path.join(os.getcwd(), "waybackup_snapshots")
+        archive.query_list(args.url, args.range, args.start, args.end, args.explicit, mode)
+        if args.list:
+            archive.print_list()
+        else:
+            archive.download_list(args.output, args.retry, args.no_redirect, args.worker)            
+            #archive.remove_empty_folders(args.output)
+    v.close()
+
+if __name__ == "__main__":
+    main()

{pywaybackup-0.6.3 → pywaybackup-0.7.1/pywaybackup.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pywaybackup
-Version: 0.6.3
+Version: 0.7.1
 Summary: Download snapshots from the Wayback Machine
 Home-page: https://github.com/bitdruid/python-wayback-machine-downloader
 Author: bitdruid
@@ -14,7 +14,7 @@ License-File: LICENSE
 
 [![PyPI](https://img.shields.io/pypi/v/pywaybackup)](https://pypi.org/project/pywaybackup/)
 [![PyPI - Downloads](https://img.shields.io/pypi/dm/pywaybackup)](https://pypi.org/project/pywaybackup/)
-![Release](https://img.shields.io/badge/Release-alpha-red)
+![Release](https://img.shields.io/badge/Release-beta-orange)
 ![Python Version](https://img.shields.io/badge/Python-3.6-blue)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 
@@ -43,6 +43,8 @@ Internet-archive is a nice source for several OSINT-information. This script is
 
 This script allows you to download content from the Wayback Machine (archive.org). You can use it to download either the latest version or all versions of web page snapshots within a specified range.
 
+<!-- ## Info -->
+
 ### Arguments
 
 - `-h`, `--help`: Show the help message and exit.
@@ -54,8 +56,8 @@ This script allows you to download content from the Wayback Machine (archive.org
 
 #### Mode Selection (Choose One)
 
-- `-c`, `--current`: Download the latest version of each file snapshot.
-- `-f`, `--full`: Download snapshots of all timestamps.
+- `-c`, `--current`: Download the latest version of each file snapshot. You will get a rebuild of the current website with all available files.
+- `-f`, `--full`: Download snapshots of all timestamps. You will get a folder per timestamp with the files available at that time.
 - `-s`, `--save`: Save a page to the Wayback Machine. (beta)
 
 #### Optional Arguments
@@ -73,6 +75,7 @@ Specify the range in years or a specific timestamp either start, end or both. If
 
 #### Additional
 
+- `--no-redirect`: Do not follow redirects of snapshots. Archive.org sometimes redirects to a different snapshot for several reasons. Downloading redirects may lead to timestamp-folders which contain some files with a different timestamp. This does not matter if you only want to download the latest version (`-c`).
 - `--verbosity [LEVEL]`: Set the verbosity: json (print json response), progress (show progress bar) or standard (default).
 - `--retry [RETRY_FAILED]`: Retry failed downloads. You can specify the number of retry attempts as an integer.
 - `--worker [AMOUNT]`: The number of worker to use for downloading (simultaneous downloads). Default is 1. Beware: Using too many worker will lead into refused connections from the Wayback Machine. Duration about 1.5 minutes.
@@ -85,8 +88,8 @@ Download latest snapshot of all files:<br>
 Download latest snapshot of all files with retries:<br>
 `waybackup -u http://example.com -c --retry 3`
 
-Download all snapshots sorted per timestamp with a specified range:<br>
-`waybackup -u http://example.com -f -r 5`
+Download all snapshots sorted per timestamp with a specified range and follow redirects:<br>
+`waybackup -u http://example.com -f -r 5 --redirect`
 
 Download all snapshots sorted per timestamp with a specified range and save to a specified folder with 3 worker:<br>
 `waybackup -u http://example.com -f -r 5 -o /home/user/Downloads/snapshots --worker 3`
@@ -94,10 +97,6 @@ Download all snapshots sorted per timestamp with a specified range and save to a
 List available snapshots per timestamp without downloading:<br>
 `waybackup -u http://example.com -f -l`
 
-## Info
-
-The script automatically follows redirects of snapshots.
-
 ## Contributing
 
 I'm always happy for some feature requests to improve the usability of this script.

{pywaybackup-0.6.3 → pywaybackup-0.7.1}/pywaybackup.egg-info/SOURCES.txt

@@ -10,6 +10,7 @@ pywaybackup/Verbosity.py
 pywaybackup/__init__.py
 pywaybackup/__version__.py
 pywaybackup/archive.py
+pywaybackup/arguments.py
 pywaybackup/main.py
 pywaybackup.egg-info/PKG-INFO
 pywaybackup.egg-info/SOURCES.txt

pywaybackup-0.7.1/requirements.txt

@@ -0,0 +1,3 @@
+requests==2.31.0
+tqdm==4.66.2
+#beautifulsoup4==4.12.3

pywaybackup-0.6.3/pywaybackup/SnapshotCollection.py

@@ -1,83 +0,0 @@
-from urllib.parse import urlparse
-import os
-
-class SnapshotCollection:
-
-    CDX_JSON = []
-    CDX_LIST = []
-
-    SNAPSHOT_COLLECTION = []
-
-    MODE_CURRENT = 0
-
-    def __init__(self):
-        pass
-
-    def create_full(self, cdxResult):
-        self.CDX_JSON = cdxResult.json()[1:]
-        self.CDX_LIST = [{"timestamp": snapshot[0], "url": snapshot[1]} for i, snapshot in enumerate(self.CDX_JSON)]
-        self.CDX_LIST = sorted(self.CDX_LIST, key=lambda k: k['timestamp'], reverse=True)
-
-    def create_current(self):
-        self.MODE_CURRENT = 1
-        self.CDX_LIST = sorted(self.CDX_LIST, key=lambda k: k['timestamp'], reverse=True)
-        cdxResult_list_filtered = []
-        url_set = set()
-        for snapshot in self.CDX_LIST:
-            if snapshot["url"] not in url_set:
-                cdxResult_list_filtered.append(snapshot)
-                url_set.add(snapshot["url"])
-        self.CDX_LIST = cdxResult_list_filtered
-
-    def create_entry(self, cdx_entry: dict, output: str) -> dict:
-        timestamp, url = cdx_entry["timestamp"], cdx_entry["url"]
-        url_type = self.__get_url_filetype(url)
-        download_url = f"http://web.archive.org/web/{timestamp}{url_type}/{url}"
-        domain, subdir, filename = self.__split_url(url)
-        if self.MODE_CURRENT: download_dir = os.path.join(output, domain, subdir)
-        else: download_dir = os.path.join(output, domain, timestamp, subdir)
-        download_file = os.path.join(download_dir, filename)
-        cdx_entry = {
-                "id": len(self.SNAPSHOT_COLLECTION),
-                "url": download_url, 
-                "file": download_file,
-                "timestamp": timestamp,
-                "origin_url": url,
-                "success": False,
-                "retry": 0
-            }
-        return cdx_entry
-
-    def count_list(self):
-        return len(self.CDX_LIST)
-    
-    def snapshot_collection_write(self, query_entry: dict):
-        if query_entry["id"] not in self.SNAPSHOT_COLLECTION:
-            self.SNAPSHOT_COLLECTION.append(query_entry)
-    
-    def snapshot_collection_update(self, id: int, key: str, value: str):
-        index = next((index for (index, d) in enumerate(self.SNAPSHOT_COLLECTION) if d["id"] == id), None)
-        if index is not None:
-            self.SNAPSHOT_COLLECTION[index][key] = value
-
-    def __get_url_filetype(self, url):
-        file_extension = os.path.splitext(url)[1][1:]
-        urltype_mapping = {
-            "jpg": "im_",
-            "jpeg": "im_",
-            "png": "im_",
-            "gif": "im_",
-            "svg": "im_",
-            "ico": "im_",
-            "css": "cs_",
-            "js": "js_"
-        }
-        urltype = urltype_mapping.get(file_extension, "id_")
-        return urltype
-    
-    def __split_url(self, url):
-        parsed_url = urlparse(url)
-        domain = parsed_url.netloc
-        subdir = parsed_url.path.strip("/").rsplit("/", 1)[0]
-        filename = parsed_url.path.split("/")[-1] or "index.html"
-        return domain, subdir, filename

pywaybackup-0.6.3/pywaybackup/__version__.py

@@ -1 +0,0 @@
-__version__ = "0.6.3"

pywaybackup-0.6.3/requirements.txt

@@ -1,2 +0,0 @@
-requests==2.31.0
-tqdm==4.66.2