pywaybackup 1.2.3__tar.gz → 1.2.4__tar.gz

{pywaybackup-1.2.3 → pywaybackup-1.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pywaybackup
-Version: 1.2.3
+Version: 1.2.4
 Summary: Download snapshots from the Wayback Machine
 Home-page: https://github.com/bitdruid/python-wayback-machine-downloader
 Author: bitdruid
@@ -26,6 +26,10 @@ Internet-archive is a nice source for several OSINT-information. This tool is a
 
 This tool allows you to download content from the Wayback Machine (archive.org). You can use it to download either the latest version or all versions of web page snapshots within a specified range.
 
+## Info
+
+Linux recommended: On windows machines, the path length is limited. It can only be overcome by editing the registry. Files which exceed the path length will not be downloaded.
+
 ## Installation
 
 ### Pip

{pywaybackup-1.2.3 → pywaybackup-1.2.4}/README.md

@@ -11,6 +11,10 @@ Internet-archive is a nice source for several OSINT-information. This tool is a
 
 This tool allows you to download content from the Wayback Machine (archive.org). You can use it to download either the latest version or all versions of web page snapshots within a specified range.
 
+## Info
+
+Linux recommended: On windows machines, the path length is limited. It can only be overcome by editing the registry. Files which exceed the path length will not be downloaded.
+
 ## Installation
 
 ### Pip

{pywaybackup-1.2.3 → pywaybackup-1.2.4}/pywaybackup/Exception.py

@@ -5,6 +5,10 @@ from datetime import datetime
 import linecache
 import traceback
 
+import re
+
+from pywaybackup.__version__ import __version__
+
 class Exception:
 
     new_debug = True
@@ -22,19 +26,20 @@ class Exception:
     @classmethod
     def exception(cls, message: str, e: Exception, tb=None):
         custom_tb = sys.exc_info()[-1] if tb is None else tb
-        original_tb = "".join(traceback.format_exception(type(e), e, e.__traceback__))
+        original_tb = cls.relativate_path("".join(traceback.format_exception(type(e), e, e.__traceback__)))
         exception_message = (
             "-------------------------\n" 
             f"!-- Exception: {message}\n"
         )
         if custom_tb is not None:
-            while custom_tb.tb_next: # loop to last traceback frame
-                custom_tb = custom_tb.tb_next 
+            while custom_tb.tb_next:  # loop to last traceback frame
+                custom_tb = custom_tb.tb_next
             tb_frame = custom_tb.tb_frame
             tb_line = custom_tb.tb_lineno
             func_name = tb_frame.f_code.co_name
-            filename = tb_frame.f_code.co_filename
+            filename = cls.relativate_path(tb_frame.f_code.co_filename)
             codeline = linecache.getline(filename, tb_line).strip()
+            local_vars = tb_frame.f_locals
             exception_message += (
                 f"!-- File: {filename}\n"
                 f"!-- Function: {func_name}\n"
@@ -45,24 +50,57 @@ class Exception:
             exception_message += "!-- Traceback is None\n"
         exception_message += (
             f"!-- Description: {e}\n"
-            "-------------------------")
+            "-------------------------"
+        )
         print(exception_message)
         if cls.debug:
             debug_file = os.path.join(cls.output, "waybackup_error.log")
             print(f"Exception log: {debug_file}")
             print("-------------------------")
             print(f"Full traceback:\n{original_tb}")
-            if cls.new_debug: # new run, overwrite file
+            if cls.new_debug:  # new run, overwrite file
                 cls.new_debug = False
                 f = open(debug_file, "w")
                 f.write("-------------------------\n")
+                f.write(f"Version: {__version__}\n")
+                f.write("-------------------------\n")
                 f.write(f"Command: {cls.command}\n")
                 f.write("-------------------------\n\n")
-            else: # current run, append to file
+            else:  # current run, append to file
                 f = open(debug_file, "a")
             f.write(datetime.now().strftime("%Y-%m-%d %H:%M:%S") + "\n")
             f.write(exception_message + "\n")
+            f.write("!-- Local Variables:\n")
+            for var_name, value in local_vars.items():
+                if var_name in ["status_message", "headers"]:
+                    continue
+                value = cls.relativate_path(str(value))
+                value = value[:666] + " ... " if len(value) > 666 else value
+                f.write(f"    -- {var_name} = {value}\n")
+            f.write("-------------------------\n")
             f.write(original_tb + "\n")
+            f.close()
+
+    @classmethod
+    def relativate_path(cls, input: str) -> str:
+        try:
+            path_pattern = re.compile(r'File "([^"]+)"')
+            if os.path.isfile(input):  # case single path
+                return os.path.relpath(input, os.getcwd())
+            input_modified = ""
+            input_lines = input.split('\n')
+            if len(input_lines) == 1: # case single line
+                return input
+            for line in input.split('\n'): # case multiple lines
+                match = path_pattern.search(line)
+                if match:
+                    original_path = match.group(1)
+                    relative_path = os.path.relpath(original_path, os.getcwd())
+                    line = line.replace(original_path, relative_path)
+                input_modified += line + "\n"
+            return input_modified
+        except ValueError:
+            return input
 
     @staticmethod
     def exception_handler(exception_type, exception, traceback):

{pywaybackup-1.2.3 → pywaybackup-1.2.4}/pywaybackup/SnapshotCollection.py

@@ -31,7 +31,13 @@ class SnapshotCollection:
     
 
     @classmethod
-    def count_list(cls):
+    def count(cls, collection=False, success=False, fail=False):
+        if collection:
+            return len(cls.SNAPSHOT_COLLECTION)
+        if success:
+            return len([entry for entry in cls.SNAPSHOT_COLLECTION if entry["file"]])
+        if fail:
+            return len([entry for entry in cls.SNAPSHOT_COLLECTION if not entry["file"]])
         return len(cls.SNAPSHOT_COLLECTION)
 
 

pywaybackup-1.2.4/pywaybackup/__version__.py

@@ -0,0 +1 @@
+__version__ = "1.2.4"

{pywaybackup-1.2.3 → pywaybackup-1.2.4}/pywaybackup/archive.py

@@ -13,7 +13,7 @@ from datetime import datetime, timezone
 
 from socket import timeout
 
-from pywaybackup.helper import url_get_timestamp, url_split, move_index, sanitize_filename
+from pywaybackup.helper import url_get_timestamp, url_split, move_index, sanitize_filename, check_nt
 
 from pywaybackup.SnapshotCollection import SnapshotCollection as sc
 
@@ -83,7 +83,7 @@ def save_page(url: str):
 
 def print_list():
     vb.write("")
-    count = sc.count_list()
+    count = sc.count(collection=True)
     if count == 0:
         vb.write("\nNo snapshots found")
     else:
@@ -104,7 +104,8 @@ def query_list(url: str, range: int, start: int, end: int, explicit: bool, mode:
             cdxResult = cdxResult.read()
             vb.write(f"\n-----> {len(cdxResult)} snapshots injected")
         except FileNotFoundError as e:
-            ex.exception("Could not inject CDX data", e)
+            vb.write("\nFILE NOT FOUND -> could not inject CDX data")
+            os._exit(1)
     else:
         vb.write("\nQuerying snapshots...")
         query_range = ""
@@ -134,7 +135,6 @@ def query_list(url: str, range: int, start: int, end: int, explicit: bool, mode:
         except requests.exceptions.ConnectionError as e:
             vb.write("\nCONNECTION REFUSED -> could not query cdx server (max retries exceeded)\n")
             os._exit(1)
-            #ex.exception("Could not query snapshots", e)
         
         if cdxbackup:
             os.makedirs(cdxbackup, exist_ok=True)
@@ -144,7 +144,7 @@ def query_list(url: str, range: int, start: int, end: int, explicit: bool, mode:
 
     cdxResult = json.loads(cdxResult)
     sc.create_list(cdxResult, mode)
-    vb.write(f"\n-----> {sc.count_list()} snapshots to utilize")
+    vb.write(f"\n-----> {sc.count(collection=True)} snapshots to utilize")
 
 
 
@@ -156,7 +156,7 @@ def download_list(output, retry, no_redirect, workers, skipset: set = None):
     """
     Download a list of urls in format: [{"timestamp": "20190815104545", "url": "https://www.google.com/"}]
     """
-    if sc.count_list() == 0: 
+    if sc.count(collection=True) == 0:
         vb.write("\nNothing to download");
         return
     vb.write("\nDownloading snapshots...", progress=0)
@@ -188,8 +188,8 @@ def download_list(output, retry, no_redirect, workers, skipset: set = None):
         thread.start()
     for thread in threads:
         thread.join()
-    successed = len([snapshot for snapshot in sc.SNAPSHOT_COLLECTION if "file" in snapshot and snapshot["file"]])
-    failed = len([snapshot for snapshot in sc.SNAPSHOT_COLLECTION if "file" in snapshot and not snapshot["file"]])
+    successed = sc.count(success=True)
+    failed = sc.count(fail=True)
     vb.write(f"\nFiles downloaded: {successed}")
     vb.write(f"Not downloaded: {failed}\n")
 
@@ -214,7 +214,7 @@ def download_loop(snapshot_queue, output, worker, retry, no_redirect, skipset=No
         while not snapshot_queue.empty():
             snapshot = snapshot_queue.get()
             status = f"\n-----> Attempt: [{attempt}/{max_attempt}] Snapshot [{sc.SNAPSHOT_COLLECTION.index(snapshot)+1}/{len(sc.SNAPSHOT_COLLECTION)}] - Worker: {worker}"
-            download_status = download(output, snapshot, connection, status, no_redirect, skipset)
+            download_status = download(output, snapshot, connection, status, no_redirect)
             if not download_status:
                 if snapshot not in failed_urls:
                     failed_urls.append(snapshot)
@@ -236,7 +236,7 @@ def download_loop(snapshot_queue, output, worker, retry, no_redirect, skipset=No
 
 
 
-def download(output, snapshot_entry, connection, status_message, no_redirect=False, skipset=None):
+def download(output, snapshot_entry, connection, status_message, no_redirect=False):
     """
     Download a single URL and save it to the specified filepath.
     If there is a redirect, the function will follow the redirect and update the download URL.
@@ -284,6 +284,15 @@ def download(output, snapshot_entry, connection, status_message, no_redirect=Fal
                 output_file = sc.create_output(download_url, snapshot_entry["timestamp"], output)
                 output_path = os.path.dirname(output_file)
 
+                # if output_file is too long for windows, skip download
+                if check_nt() and len(output_file) > 255:
+                    status_message = f"{status_message}\n" + \
+                        f"PATH TOO LONG TO SAVE FILE -> HTTP: {response_status} - {response_status_message}\n" + \
+                        f"                           -> URL: {download_url}"
+                    sc.snapshot_entry_modify(snapshot_entry, "file", "PATH TOO LONG TO SAVE FILE")
+                    vb.write(status_message)
+                    return True
+
                 # case if output_path is a file, move file to temporary name, create output_path and move file into output_path
                 if os.path.isfile(output_path):
                     move_index(existpath=output_path)
@@ -293,14 +302,6 @@ def download(output, snapshot_entry, connection, status_message, no_redirect=Fal
                 if os.path.isdir(output_file):
                     output_file = move_index(existfile=output_file, filebuffer=response_data)
 
-                # if filename is too long, skip download
-                if len(os.path.basename(output_file)) > 255:
-                    status_message = f"{status_message}\n" + \
-                        f"FILENAME TOO LONG -> HTTP: {response_status} - {response_status_message}\n" + \
-                        f"                  -> URL: {download_url}"
-                    vb.write(status_message)
-                    return True
-
                 if not os.path.isfile(output_file):
                     with open(output_file, 'wb') as file:
                         if response.getheader('Content-Encoding') == 'gzip':
@@ -378,7 +379,7 @@ def csv_close(csv_path: str, url: str):
     """
     try:
         csv_path = csv_filepath(csv_path, url)
-        if sc.count_list() > 0:
+        if sc.count(collection=True) > 0:
             if os.path.exists(csv_path): # append to existing file
                 existing_rows = set()
                 with open(csv_path, mode='r') as file: # read existing rows
@@ -425,11 +426,13 @@ def skip_open(csv_path: str, url: str) -> tuple:
     """
     try:
         csv_path = csv_filepath(csv_path, url)
-        if os.path.exists(csv_path) and os.path.getsize(csv_path) > 0:
+        if os.path.isfile(csv_path) and os.path.getsize(csv_path) > 0:
             csv_file = open(csv_path, mode='r')
             skipset = set(csv_read(csv_file))
+            csv_file.close()
             return skipset
         else:
+            vb.write("\nNo CSV-file or content found to load skipable URLs")
             return None
     except Exception as e:
         ex.exception("Could not open CSV-file", e)

{pywaybackup-1.2.3 → pywaybackup-1.2.4}/pywaybackup/helper.py

@@ -3,6 +3,14 @@ import os
 import shutil
 import magic
 
+
+def check_nt():
+    """
+    Check if the OS is Windows.
+    """
+    return os.name == "nt"
+
+
 def sanitize_filename(input: str) -> str:
     """
     Sanitize a string to be used as (part of) a filename.
@@ -21,6 +29,7 @@ def url_get_timestamp(url):
         timestamp = url.split("id_/")[0].split("/")[-1]
         return timestamp
 
+
 def url_split(url, index=False):
     """
     Split a URL into domain, subdir, and filename.
@@ -38,7 +47,7 @@ def url_split(url, index=False):
         filename = "index.html" if index else ""
     subdir = "/".join(path_parts).strip("/")
     # sanitize subdir and filename for windows
-    if os.name == "nt":
+    if check_nt():
         special_chars = [":", "*", "?", "&", "=", "<", ">", "\\", "|"]
         for char in special_chars:
             subdir = subdir.replace(char, f"%{ord(char):02x}")
@@ -46,6 +55,7 @@ def url_split(url, index=False):
     filename = filename.replace("%20", " ")
     return domain, subdir, filename
 
+
 def move_index(existpath: str = None, existfile: str = None, filebuffer: bytes = None):
     """
     1. If existpath is given but can't be created because a file exists with the same name
@@ -73,6 +83,7 @@ def move_index(existpath: str = None, existfile: str = None, filebuffer: bytes =
             else:
                 return os.path.join(existfile, "index.html")
     
+
 def check_index_mime(filebuffer: bytes) -> bool:
     mime = magic.Magic(mime=True)
     mime_type = mime.from_buffer(filebuffer)

{pywaybackup-1.2.3 → pywaybackup-1.2.4}/pywaybackup/main.py

@@ -14,6 +14,8 @@ def main():
     if args.output is None:
         args.output = os.path.join(os.getcwd(), "waybackup_snapshots")
         os.makedirs(args.output, exist_ok=True)
+    else:
+        os.makedirs(args.output, exist_ok=True)
 
     ex.init(args.debug, args.output, command)
     vb.init(args.verbosity)

{pywaybackup-1.2.3 → pywaybackup-1.2.4}/pywaybackup.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pywaybackup
-Version: 1.2.3
+Version: 1.2.4
 Summary: Download snapshots from the Wayback Machine
 Home-page: https://github.com/bitdruid/python-wayback-machine-downloader
 Author: bitdruid
@@ -26,6 +26,10 @@ Internet-archive is a nice source for several OSINT-information. This tool is a
 
 This tool allows you to download content from the Wayback Machine (archive.org). You can use it to download either the latest version or all versions of web page snapshots within a specified range.
 
+## Info
+
+Linux recommended: On windows machines, the path length is limited. It can only be overcome by editing the registry. Files which exceed the path length will not be downloaded.
+
 ## Installation
 
 ### Pip

pywaybackup-1.2.3/pywaybackup/__version__.py

@@ -1 +0,0 @@
-__version__ = "1.2.3"