pywaybackup 0.2.6__tar.gz → 0.3.0__tar.gz

{pywaybackup-0.2.6 → pywaybackup-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pywaybackup
-Version: 0.2.6
+Version: 0.3.0
 Summary: Download snapshots from the Wayback Machine
 Home-page: https://github.com/bitdruid/waybackup
 Author: bitdruid
@@ -12,22 +12,15 @@ License-File: LICENSE
 
 # archive wayback downloader
 
-![Version](https://img.shields.io/badge/Version-0.2.6-blue)
+![Version](https://img.shields.io/badge/Version-0.3.0-blue)
 ![Release](https://img.shields.io/badge/Release-alpha-red)
+![Python Version](https://img.shields.io/badge/Python-3.6-blue)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 
 Downloading archived web pages from the [Wayback Machine](https://archive.org/web/).
 
 Internet-archive is a nice source for several OSINT-information. This script is a work in progress to query and fetch archived web pages.
 
-This project is not intended to get fast results. Moreover it is a tool to get a lot of data over a long period of time.
-
-## Limitations
-
-The wayback-machine does refuse connections over public access if the query rate is too high. So for now there seems no possibility to implement a multi-threaded download. As soon as a connection is refused, the script will wait and retry the query. Existing projects seem to ignore this limitation and just rush through the queries. This resulted in a lot of missing files and probably missing knowledge about the target.
-
-Timeout seems to be about 2.5 minutes per 20 downloads.
-
 ## Installation
 
 ### Pip
@@ -41,10 +34,8 @@ Timeout seems to be about 2.5 minutes per 20 downloads.
 
 1. Clone the repository <br>
    ```git clone https://github.com/bitdruid/waybackup.git```
-2. Install requirements <br>
-   ```pip install -r requirements.txt```
-3. Run the script <br>
-   ```python waybackup.py -h```
+2. Install <br>
+   ```python setup.py install```
 
 ## Usage
 

{pywaybackup-0.2.6 → pywaybackup-0.3.0}/README.md

@@ -1,21 +1,14 @@
 # archive wayback downloader
 
-![Version](https://img.shields.io/badge/Version-0.2.6-blue)
+![Version](https://img.shields.io/badge/Version-0.3.0-blue)
 ![Release](https://img.shields.io/badge/Release-alpha-red)
+![Python Version](https://img.shields.io/badge/Python-3.6-blue)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 
 Downloading archived web pages from the [Wayback Machine](https://archive.org/web/).
 
 Internet-archive is a nice source for several OSINT-information. This script is a work in progress to query and fetch archived web pages.
 
-This project is not intended to get fast results. Moreover it is a tool to get a lot of data over a long period of time.
-
-## Limitations
-
-The wayback-machine does refuse connections over public access if the query rate is too high. So for now there seems no possibility to implement a multi-threaded download. As soon as a connection is refused, the script will wait and retry the query. Existing projects seem to ignore this limitation and just rush through the queries. This resulted in a lot of missing files and probably missing knowledge about the target.
-
-Timeout seems to be about 2.5 minutes per 20 downloads.
-
 ## Installation
 
 ### Pip
@@ -29,10 +22,8 @@ Timeout seems to be about 2.5 minutes per 20 downloads.
 
 1. Clone the repository <br>
    ```git clone https://github.com/bitdruid/waybackup.git```
-2. Install requirements <br>
-   ```pip install -r requirements.txt```
-3. Run the script <br>
-   ```python waybackup.py -h```
+2. Install <br>
+   ```python setup.py install```
 
 ## Usage
 

pywaybackup-0.3.0/pywaybackup/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.3.0"

{pywaybackup-0.2.6 → pywaybackup-0.3.0}/pywaybackup/archive.py

@@ -1,11 +1,14 @@
-import threading
+#import threading
+from pathlib import Path
 import requests
 import datetime
 import os
-#import magic
+import magic
+import threading
 from pprint import pprint
 import time
 import pathlib
+import http.client
 
 def print_result(result_list):
     print("")
@@ -121,13 +124,11 @@ def remove_empty_folders(path, remove_root=True):
 # example download: http://web.archive.org/web/20190815104545id_/https://www.google.com/
 # example url: https://www.google.com/
 # example timestamp: 20190815104545
-def download_url_list(cdxResult_list, output, retry, mode):
+def download_prepare_list(cdxResult_list, output, retry, workers, mode):
     """
     Download a list of urls in format: [{"timestamp": "20190815104545", "url": "https://www.google.com/"}]
     """
-    #def download_batch(cdxResult_list):
     print("\nDownloading latest snapshots of each file...")
-    failed_urls = []
     download_list = []
     for snapshot in cdxResult_list:
         timestamp, url = snapshot["timestamp"], snapshot["url"]
@@ -136,29 +137,34 @@ def download_url_list(cdxResult_list, output, retry, mode):
         domain, subdir, filename = split_url(url)
         if mode == "current": download_dir = os.path.join(output, domain, subdir)
         if mode == "full": download_dir = os.path.join(output, domain, timestamp, subdir)
-        download_filepath = os.path.join(download_dir, filename)
         download_list.append({"url": download_url, "filename": filename, "filepath": download_dir})
-    # download urls
-    for download_entry in download_list:
-        print(f"\n-----> Snapshot [{download_list.index(download_entry) + 1}/{len(download_list)}]")
-        download_url, download_filename, download_filepath = download_entry["url"], download_entry["filename"], download_entry["filepath"]
-        download_status=download_url_entry(download_url, download_filename, download_filepath)
+    if workers > 1:
+        print(f"\n-----> Simultaneous downloads: {workers}")
+    threads = []
+    worker = 0
+    batch_size = len(download_list) // workers + 1
+    batch_list = [download_list[i:i + batch_size] for i in range(0, len(download_list), batch_size)]
+    for batch in batch_list:
+        worker += 1
+        thread = threading.Thread(target=download_url_list, args=(batch, worker, retry))
+        threads.append(thread)
+        thread.start()
+    for thread in threads:
+        thread.join()
+
+def download_url_list(url_list, worker, retry):
+    failed_urls = []
+    connection = http.client.HTTPSConnection("web.archive.org")
+    for url_entry in url_list:
+        status = f"\n-----> Snapshot [{url_list.index(url_entry) + 1}/{len(url_list)}] Worker: {worker}"
+        download_url, download_filename, download_filepath = url_entry["url"], url_entry["filename"], url_entry["filepath"]
+        download_status=download_url_entry(download_url, download_filename, download_filepath, connection, status)
         if download_status != bool(1): failed_urls.append({"url": download_url, "filename": download_filename, "filepath": download_filepath})
-    if retry > 0 or retry is True:
-        print(f"\n-----> Fail downloads: {len(failed_urls)}")
-        download_retry(failed_urls, retry)
-    
-    # batch_size = len(download_list) // 10
-    # batch_list = [download_list[i:i + batch_size] for i in range(0, len(download_list), batch_size)]
-    # for batch in batch_list:
-    #     threads = []
-    #     thread = threading.Thread(target=download_batch, args=(batch,))
-    #     thread.start()
-    #     threads.append(thread)
-    # for thread in threads:
-    #     thread.join()
-
-def download_retry(failed_urls, retry):
+    if retry:
+        download_retry(failed_urls, retry, connection)
+    connection.close()
+
+def download_retry(failed_urls, retry, connection):
     """
     Retry failed downloads.
     failed_urls: [{"url": download_url, "filename": download_filename, "filepath": download_filepath}]
@@ -170,50 +176,60 @@ def download_retry(failed_urls, retry):
         print(f"\n-----> Retrying...")
         retry_urls = []
         for failed_entry in failed_urls:
+            status = f"\n-----> RETRY attempt: [{attempt}/{max_attempt}] Snapshot [{failed_urls.index(failed_entry) + 1}/{len(failed_urls)}]"
             download_url, download_filename, download_filepath = failed_entry["url"], failed_entry["filename"], failed_entry["filepath"]
-            print(f"\n-----> RETRY attempt: [{attempt}/{max_attempt}] Snapshot [{failed_urls.index(failed_entry) + 1}/{len(failed_urls)}]")
-            retry_status=download_url_entry(download_url, download_filename, download_filepath)
+            retry_status=download_url_entry(download_url, download_filename, download_filepath, connection, status)
             if retry_status != bool(1):
                 retry_urls.append({"url": download_url, "filename": download_filename, "filepath": download_filepath})
         failed_urls = retry_urls
         print(f"\n-----> Fail downloads: {len(failed_urls)}")
         if retry != None: attempt += 1
 
-def download_url_entry(url, filename, filepath):
+def download_url_entry(url, filename, filepath, connection, status):
     """
     Download a single url.
     Success: return bool(1)
-    Fail: return url
+    Fail: return bool(0)
     """
-    # create output dirs
     create_dirs(filepath)
     output = os.path.join(filepath, filename)
     max_retries = 2
-    sleep_time = 40
+    sleep_time = 45
     headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36'}
-    # download url
     for i in range(max_retries):
         try:
-            data = requests.get(url, headers=headers)
+            connection.request("GET", url, headers=headers)
+            response = connection.getresponse()
+            data = response.read()
             with open(output, 'wb') as file:
-                file.write(data.content)
+                file.write(data)
+            print(status)
             print(f"SUCCESS -> {url}")
             print(f"        -> {output}")
             return bool(1)
-        except requests.exceptions.ConnectionError as e:
+        except http.client.HTTPException as e:
+            print(status)
             print(f"REFUSED -> ({i+1}/{max_retries}), reconnect in {sleep_time} seconds...")
             time.sleep(sleep_time)
-    else:
-        print(f"FAILED  -> download, append to failed_urls: {url}")
-        return bool(0)
-
-
+    print(f"FAILED  -> download, append to failed_urls: {url}")
+    return bool(0)
 
 
 
 # scan output folder and guess mimetype for each file
 # if add file extension if not present
-# def guess_mimetype(filepath):
-#     print("")
-#     print("Guessing mimetypes for unknown files...")
-    
+# def detect_filetype(filepath):
+#     print("\nDetecting filetypes...")
+#     path = Path(filepath)
+#     if not path.is_dir():
+#         print(f"\n-----> ERROR: {filepath} is not a directory"); return
+#     for file_path in path.rglob("*"):
+#         if file_path.is_file():
+#             file_extension = file_path.suffix
+#             if not file_extension:
+#                 mime_type = magic.from_file(str(file_path), mime=True)
+#                 file_extension = mime_type.split("/")[-1]
+#                 new_file_path = file_path.with_suffix('.' + file_extension)
+#                 file_path.rename(new_file_path)
+#                 print(f"NO EXT -> {file_path}")
+#                 print(f"   NEW -> {new_file_path}")

pywaybackup-0.2.6/pywaybackup/pywaybackup.py → pywaybackup-0.3.0/pywaybackup/main.py

@@ -1,7 +1,8 @@
 import pywaybackup.archive as archive 
 import argparse
 import os
-__version__ = "0.2.6"
+
+from pywaybackup.__version__ import __version__
 
 def main():
     parser = argparse.ArgumentParser(description='Download from wayback machine (archive.org)')
@@ -16,8 +17,9 @@ def main():
     optional.add_argument('-r', '--range', type=int, help='Range in years to search')
     optional.add_argument('-o', '--output', type=str, help='Output folder')
     special = parser.add_argument_group('special')
-    #special.add_argument('--detect-filetype', action='store_true', help='If a file has no extension, try to detect the filetype')
+    special.add_argument('--detect-filetype', action='store_true', help='If a file has no extension, try to detect the filetype')
     special.add_argument('--retry-failed', nargs='?', const=True, type=int, help='Retry failed downloads (opt tries as int, else infinite)')
+    special.add_argument('--workers', type=int, default=1, help='Number of workers (simultaneous downloads)')
 
     args = parser.parse_args()
     if args.current:
@@ -31,8 +33,10 @@ def main():
     if args.list:
         archive.print_result(cdxResult_list)
     if not args.list:
-        archive.download_url_list(cdxResult_list, args.output, args.retry_failed, mode)
+        archive.download_prepare_list(cdxResult_list, args.output, args.retry_failed, args.workers, mode)
         archive.remove_empty_folders(args.output)
+    if args.detect_filetype:
+        archive.detect_filetype(args.output)
 
 if __name__ == "__main__":
     main()

{pywaybackup-0.2.6 → pywaybackup-0.3.0}/pywaybackup.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pywaybackup
-Version: 0.2.6
+Version: 0.3.0
 Summary: Download snapshots from the Wayback Machine
 Home-page: https://github.com/bitdruid/waybackup
 Author: bitdruid
@@ -12,22 +12,15 @@ License-File: LICENSE
 
 # archive wayback downloader
 
-![Version](https://img.shields.io/badge/Version-0.2.6-blue)
+![Version](https://img.shields.io/badge/Version-0.3.0-blue)
 ![Release](https://img.shields.io/badge/Release-alpha-red)
+![Python Version](https://img.shields.io/badge/Python-3.6-blue)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 
 Downloading archived web pages from the [Wayback Machine](https://archive.org/web/).
 
 Internet-archive is a nice source for several OSINT-information. This script is a work in progress to query and fetch archived web pages.
 
-This project is not intended to get fast results. Moreover it is a tool to get a lot of data over a long period of time.
-
-## Limitations
-
-The wayback-machine does refuse connections over public access if the query rate is too high. So for now there seems no possibility to implement a multi-threaded download. As soon as a connection is refused, the script will wait and retry the query. Existing projects seem to ignore this limitation and just rush through the queries. This resulted in a lot of missing files and probably missing knowledge about the target.
-
-Timeout seems to be about 2.5 minutes per 20 downloads.
-
 ## Installation
 
 ### Pip
@@ -41,10 +34,8 @@ Timeout seems to be about 2.5 minutes per 20 downloads.
 
 1. Clone the repository <br>
    ```git clone https://github.com/bitdruid/waybackup.git```
-2. Install requirements <br>
-   ```pip install -r requirements.txt```
-3. Run the script <br>
-   ```python waybackup.py -h```
+2. Install <br>
+   ```python setup.py install```
 
 ## Usage
 

{pywaybackup-0.2.6 → pywaybackup-0.3.0}/pywaybackup.egg-info/SOURCES.txt

@@ -2,11 +2,13 @@ LICENSE
 README.md
 setup.py
 pywaybackup/__init__.py
+pywaybackup/__version__.py
 pywaybackup/archive.py
-pywaybackup/pywaybackup.py
+pywaybackup/main.py
 pywaybackup.egg-info/PKG-INFO
 pywaybackup.egg-info/SOURCES.txt
 pywaybackup.egg-info/dependency_links.txt
 pywaybackup.egg-info/entry_points.txt
 pywaybackup.egg-info/requires.txt
-pywaybackup.egg-info/top_level.txt
+pywaybackup.egg-info/top_level.txt
+test/test.py

pywaybackup-0.3.0/pywaybackup.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+waybackup = pywaybackup.main:main

pywaybackup-0.3.0/pywaybackup.egg-info/requires.txt

@@ -0,0 +1,2 @@
+python-magic==0.4.27
+requests==2.31.0

{pywaybackup-0.2.6 → pywaybackup-0.3.0}/setup.py

@@ -1,7 +1,7 @@
 
 from setuptools import setup, find_packages
+from pywaybackup.__version__ import __version__ as VERSION
 
-VERSION = '0.2.6'
 DESCRIPTION = 'Download snapshots from the Wayback Machine'
 
 import pkg_resources
@@ -21,7 +21,7 @@ setup(
     install_requires=parse_requirements('./requirements.txt'),
     entry_points={
         'console_scripts': [
-            'waybackup = pywaybackup.pywaybackup:main',
+            'waybackup = pywaybackup.main:main',
         ],
     },
     author='bitdruid',

pywaybackup-0.3.0/test/test.py

@@ -0,0 +1,25 @@
+def dl(url: str):
+    from selenium import webdriver
+    from bs4 import BeautifulSoup
+    browser = webdriver.Chrome()
+    options = webdriver.ChromeOptions()
+    options.add_argument('--headless=new')
+    options.add_argument('--disable-gpu')
+    options.add_argument('--no-sandbox')
+    options.add_argument('--disable-dev-shm-usage')
+    browser = webdriver.Chrome(options=options)
+    browser.get(url)
+    soup = BeautifulSoup(browser.page_source, 'html.parser')
+    target_path = "test.html"
+    with open (target_path, "w") as file:
+        file.write(soup.prettify())
+    print(soup.prettify())
+
+if __name__ == "__main__":
+    import sys
+    if sys.argv is None or len(sys.argv) < 2:
+        print("No url given")
+        sys.exit()
+    url = sys.argv[1]
+    dl(url)
+

pywaybackup-0.2.6/pywaybackup.egg-info/entry_points.txt

@@ -1,2 +0,0 @@
-[console_scripts]
-waybackup = pywaybackup.pywaybackup:main

pywaybackup-0.2.6/pywaybackup.egg-info/requires.txt

@@ -1 +0,0 @@
-requests==2.31.0