pywaybackup 0.2.6__tar.gz → 0.2.7__tar.gz

{pywaybackup-0.2.6 → pywaybackup-0.2.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pywaybackup
-Version: 0.2.6
+Version: 0.2.7
 Summary: Download snapshots from the Wayback Machine
 Home-page: https://github.com/bitdruid/waybackup
 Author: bitdruid
@@ -12,22 +12,15 @@ License-File: LICENSE
 
 # archive wayback downloader
 
-![Version](https://img.shields.io/badge/Version-0.2.6-blue)
+![Version](https://img.shields.io/badge/Version-0.2.7-blue)
 ![Release](https://img.shields.io/badge/Release-alpha-red)
+![Python Version](https://img.shields.io/badge/Python-3.6-blue)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 
 Downloading archived web pages from the [Wayback Machine](https://archive.org/web/).
 
 Internet-archive is a nice source for several OSINT-information. This script is a work in progress to query and fetch archived web pages.
 
-This project is not intended to get fast results. Moreover it is a tool to get a lot of data over a long period of time.
-
-## Limitations
-
-The wayback-machine does refuse connections over public access if the query rate is too high. So for now there seems no possibility to implement a multi-threaded download. As soon as a connection is refused, the script will wait and retry the query. Existing projects seem to ignore this limitation and just rush through the queries. This resulted in a lot of missing files and probably missing knowledge about the target.
-
-Timeout seems to be about 2.5 minutes per 20 downloads.
-
 ## Installation
 
 ### Pip
@@ -41,10 +34,8 @@ Timeout seems to be about 2.5 minutes per 20 downloads.
 
 1. Clone the repository <br>
    ```git clone https://github.com/bitdruid/waybackup.git```
-2. Install requirements <br>
-   ```pip install -r requirements.txt```
-3. Run the script <br>
-   ```python waybackup.py -h```
+2. Install <br>
+   ```python setup.py install```
 
 ## Usage
 

{pywaybackup-0.2.6 → pywaybackup-0.2.7}/README.md

@@ -1,21 +1,14 @@
 # archive wayback downloader
 
-![Version](https://img.shields.io/badge/Version-0.2.6-blue)
+![Version](https://img.shields.io/badge/Version-0.2.7-blue)
 ![Release](https://img.shields.io/badge/Release-alpha-red)
+![Python Version](https://img.shields.io/badge/Python-3.6-blue)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 
 Downloading archived web pages from the [Wayback Machine](https://archive.org/web/).
 
 Internet-archive is a nice source for several OSINT-information. This script is a work in progress to query and fetch archived web pages.
 
-This project is not intended to get fast results. Moreover it is a tool to get a lot of data over a long period of time.
-
-## Limitations
-
-The wayback-machine does refuse connections over public access if the query rate is too high. So for now there seems no possibility to implement a multi-threaded download. As soon as a connection is refused, the script will wait and retry the query. Existing projects seem to ignore this limitation and just rush through the queries. This resulted in a lot of missing files and probably missing knowledge about the target.
-
-Timeout seems to be about 2.5 minutes per 20 downloads.
-
 ## Installation
 
 ### Pip
@@ -29,10 +22,8 @@ Timeout seems to be about 2.5 minutes per 20 downloads.
 
 1. Clone the repository <br>
    ```git clone https://github.com/bitdruid/waybackup.git```
-2. Install requirements <br>
-   ```pip install -r requirements.txt```
-3. Run the script <br>
-   ```python waybackup.py -h```
+2. Install <br>
+   ```python setup.py install```
 
 ## Usage
 

pywaybackup-0.2.7/pywaybackup/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.2.7"

{pywaybackup-0.2.6 → pywaybackup-0.2.7}/pywaybackup/archive.py

@@ -1,11 +1,13 @@
-import threading
+#import threading
+from pathlib import Path
 import requests
 import datetime
 import os
-#import magic
+import magic
 from pprint import pprint
 import time
 import pathlib
+import http.client
 
 def print_result(result_list):
     print("")
@@ -129,6 +131,7 @@ def download_url_list(cdxResult_list, output, retry, mode):
     print("\nDownloading latest snapshots of each file...")
     failed_urls = []
     download_list = []
+    connection = http.client.HTTPSConnection("web.archive.org")
     for snapshot in cdxResult_list:
         timestamp, url = snapshot["timestamp"], snapshot["url"]
         type = determine_url_filetype(url)
@@ -142,11 +145,12 @@ def download_url_list(cdxResult_list, output, retry, mode):
     for download_entry in download_list:
         print(f"\n-----> Snapshot [{download_list.index(download_entry) + 1}/{len(download_list)}]")
         download_url, download_filename, download_filepath = download_entry["url"], download_entry["filename"], download_entry["filepath"]
-        download_status=download_url_entry(download_url, download_filename, download_filepath)
+        download_status=download_url_entry(download_url, download_filename, download_filepath, connection)
         if download_status != bool(1): failed_urls.append({"url": download_url, "filename": download_filename, "filepath": download_filepath})
-    if retry > 0 or retry is True:
+    if retry:
         print(f"\n-----> Fail downloads: {len(failed_urls)}")
-        download_retry(failed_urls, retry)
+        download_retry(failed_urls, retry, connection)
+    connection.close()
     
     # batch_size = len(download_list) // 10
     # batch_list = [download_list[i:i + batch_size] for i in range(0, len(download_list), batch_size)]
@@ -158,7 +162,7 @@ def download_url_list(cdxResult_list, output, retry, mode):
     # for thread in threads:
     #     thread.join()
 
-def download_retry(failed_urls, retry):
+def download_retry(failed_urls, retry, connection):
     """
     Retry failed downloads.
     failed_urls: [{"url": download_url, "filename": download_filename, "filepath": download_filepath}]
@@ -172,48 +176,56 @@ def download_retry(failed_urls, retry):
         for failed_entry in failed_urls:
             download_url, download_filename, download_filepath = failed_entry["url"], failed_entry["filename"], failed_entry["filepath"]
             print(f"\n-----> RETRY attempt: [{attempt}/{max_attempt}] Snapshot [{failed_urls.index(failed_entry) + 1}/{len(failed_urls)}]")
-            retry_status=download_url_entry(download_url, download_filename, download_filepath)
+            retry_status=download_url_entry(download_url, download_filename, download_filepath, connection)
             if retry_status != bool(1):
                 retry_urls.append({"url": download_url, "filename": download_filename, "filepath": download_filepath})
         failed_urls = retry_urls
         print(f"\n-----> Fail downloads: {len(failed_urls)}")
         if retry != None: attempt += 1
 
-def download_url_entry(url, filename, filepath):
+def download_url_entry(url, filename, filepath, connection):
     """
     Download a single url.
     Success: return bool(1)
-    Fail: return url
+    Fail: return bool(0)
     """
-    # create output dirs
     create_dirs(filepath)
     output = os.path.join(filepath, filename)
     max_retries = 2
-    sleep_time = 40
+    sleep_time = 45
     headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36'}
-    # download url
     for i in range(max_retries):
         try:
-            data = requests.get(url, headers=headers)
+            connection.request("GET", url, headers=headers)
+            response = connection.getresponse()
+            data = response.read()
             with open(output, 'wb') as file:
-                file.write(data.content)
+                file.write(data)
             print(f"SUCCESS -> {url}")
             print(f"        -> {output}")
             return bool(1)
-        except requests.exceptions.ConnectionError as e:
+        except http.client.HTTPException as e:
             print(f"REFUSED -> ({i+1}/{max_retries}), reconnect in {sleep_time} seconds...")
             time.sleep(sleep_time)
-    else:
-        print(f"FAILED  -> download, append to failed_urls: {url}")
-        return bool(0)
-
-
+    print(f"FAILED  -> download, append to failed_urls: {url}")
+    return bool(0)
 
 
 
 # scan output folder and guess mimetype for each file
 # if add file extension if not present
-# def guess_mimetype(filepath):
-#     print("")
-#     print("Guessing mimetypes for unknown files...")
-    
+# def detect_filetype(filepath):
+#     print("\nDetecting filetypes...")
+#     path = Path(filepath)
+#     if not path.is_dir():
+#         print(f"\n-----> ERROR: {filepath} is not a directory"); return
+#     for file_path in path.rglob("*"):
+#         if file_path.is_file():
+#             file_extension = file_path.suffix
+#             if not file_extension:
+#                 mime_type = magic.from_file(str(file_path), mime=True)
+#                 file_extension = mime_type.split("/")[-1]
+#                 new_file_path = file_path.with_suffix('.' + file_extension)
+#                 file_path.rename(new_file_path)
+#                 print(f"NO EXT -> {file_path}")
+#                 print(f"   NEW -> {new_file_path}")

pywaybackup-0.2.6/pywaybackup/pywaybackup.py → pywaybackup-0.2.7/pywaybackup/main.py

@@ -1,7 +1,8 @@
 import pywaybackup.archive as archive 
 import argparse
 import os
-__version__ = "0.2.6"
+
+from pywaybackup.__version__ import __version__
 
 def main():
     parser = argparse.ArgumentParser(description='Download from wayback machine (archive.org)')
@@ -16,7 +17,7 @@ def main():
     optional.add_argument('-r', '--range', type=int, help='Range in years to search')
     optional.add_argument('-o', '--output', type=str, help='Output folder')
     special = parser.add_argument_group('special')
-    #special.add_argument('--detect-filetype', action='store_true', help='If a file has no extension, try to detect the filetype')
+    special.add_argument('--detect-filetype', action='store_true', help='If a file has no extension, try to detect the filetype')
     special.add_argument('--retry-failed', nargs='?', const=True, type=int, help='Retry failed downloads (opt tries as int, else infinite)')
 
     args = parser.parse_args()
@@ -33,6 +34,8 @@ def main():
     if not args.list:
         archive.download_url_list(cdxResult_list, args.output, args.retry_failed, mode)
         archive.remove_empty_folders(args.output)
+    if args.detect_filetype:
+        archive.detect_filetype(args.output)
 
 if __name__ == "__main__":
     main()

{pywaybackup-0.2.6 → pywaybackup-0.2.7}/pywaybackup.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pywaybackup
-Version: 0.2.6
+Version: 0.2.7
 Summary: Download snapshots from the Wayback Machine
 Home-page: https://github.com/bitdruid/waybackup
 Author: bitdruid
@@ -12,22 +12,15 @@ License-File: LICENSE
 
 # archive wayback downloader
 
-![Version](https://img.shields.io/badge/Version-0.2.6-blue)
+![Version](https://img.shields.io/badge/Version-0.2.7-blue)
 ![Release](https://img.shields.io/badge/Release-alpha-red)
+![Python Version](https://img.shields.io/badge/Python-3.6-blue)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 
 Downloading archived web pages from the [Wayback Machine](https://archive.org/web/).
 
 Internet-archive is a nice source for several OSINT-information. This script is a work in progress to query and fetch archived web pages.
 
-This project is not intended to get fast results. Moreover it is a tool to get a lot of data over a long period of time.
-
-## Limitations
-
-The wayback-machine does refuse connections over public access if the query rate is too high. So for now there seems no possibility to implement a multi-threaded download. As soon as a connection is refused, the script will wait and retry the query. Existing projects seem to ignore this limitation and just rush through the queries. This resulted in a lot of missing files and probably missing knowledge about the target.
-
-Timeout seems to be about 2.5 minutes per 20 downloads.
-
 ## Installation
 
 ### Pip
@@ -41,10 +34,8 @@ Timeout seems to be about 2.5 minutes per 20 downloads.
 
 1. Clone the repository <br>
    ```git clone https://github.com/bitdruid/waybackup.git```
-2. Install requirements <br>
-   ```pip install -r requirements.txt```
-3. Run the script <br>
-   ```python waybackup.py -h```
+2. Install <br>
+   ```python setup.py install```
 
 ## Usage
 

{pywaybackup-0.2.6 → pywaybackup-0.2.7}/pywaybackup.egg-info/SOURCES.txt

@@ -2,11 +2,13 @@ LICENSE
 README.md
 setup.py
 pywaybackup/__init__.py
+pywaybackup/__version__.py
 pywaybackup/archive.py
-pywaybackup/pywaybackup.py
+pywaybackup/main.py
 pywaybackup.egg-info/PKG-INFO
 pywaybackup.egg-info/SOURCES.txt
 pywaybackup.egg-info/dependency_links.txt
 pywaybackup.egg-info/entry_points.txt
 pywaybackup.egg-info/requires.txt
-pywaybackup.egg-info/top_level.txt
+pywaybackup.egg-info/top_level.txt
+test/test.py

pywaybackup-0.2.7/pywaybackup.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+waybackup = pywaybackup.main:main

pywaybackup-0.2.7/pywaybackup.egg-info/requires.txt

@@ -0,0 +1,2 @@
+python-magic==0.4.27
+requests==2.31.0

{pywaybackup-0.2.6 → pywaybackup-0.2.7}/setup.py

@@ -1,7 +1,7 @@
 
 from setuptools import setup, find_packages
+from pywaybackup.__version__ import __version__ as VERSION
 
-VERSION = '0.2.6'
 DESCRIPTION = 'Download snapshots from the Wayback Machine'
 
 import pkg_resources
@@ -21,7 +21,7 @@ setup(
     install_requires=parse_requirements('./requirements.txt'),
     entry_points={
         'console_scripts': [
-            'waybackup = pywaybackup.pywaybackup:main',
+            'waybackup = pywaybackup.main:main',
         ],
     },
     author='bitdruid',

pywaybackup-0.2.7/test/test.py

@@ -0,0 +1,25 @@
+def dl(url: str):
+    from selenium import webdriver
+    from bs4 import BeautifulSoup
+    browser = webdriver.Chrome()
+    options = webdriver.ChromeOptions()
+    options.add_argument('--headless=new')
+    options.add_argument('--disable-gpu')
+    options.add_argument('--no-sandbox')
+    options.add_argument('--disable-dev-shm-usage')
+    browser = webdriver.Chrome(options=options)
+    browser.get(url)
+    soup = BeautifulSoup(browser.page_source, 'html.parser')
+    target_path = "test.html"
+    with open (target_path, "w") as file:
+        file.write(soup.prettify())
+    print(soup.prettify())
+
+if __name__ == "__main__":
+    import sys
+    if sys.argv is None or len(sys.argv) < 2:
+        print("No url given")
+        sys.exit()
+    url = sys.argv[1]
+    dl(url)
+

pywaybackup-0.2.6/pywaybackup.egg-info/entry_points.txt

@@ -1,2 +0,0 @@
-[console_scripts]
-waybackup = pywaybackup.pywaybackup:main

pywaybackup-0.2.6/pywaybackup.egg-info/requires.txt

@@ -1 +0,0 @@
-requests==2.31.0