pywaybackup 1.3.2__tar.gz → 1.4.0__tar.gz

{pywaybackup-1.3.2 → pywaybackup-1.4.0}/.github/ISSUE_TEMPLATE/bug.md

@@ -1,12 +1,14 @@
 ---
 name: Bug
 about: Describe a faulty behavior
-title: ''
+title: "[BUG]"
 labels: bug
 assignees: ''
 
 ---
 
+<ins><b>Please select the according OS-Label</b></ins>
+
 **Describe**
 Describe the bug / unexpected event
 

{pywaybackup-1.3.2/pywaybackup.egg-info → pywaybackup-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pywaybackup
-Version: 1.3.2
+Version: 1.4.0
 Summary: Download snapshots from the Wayback Machine
 Home-page: https://github.com/bitdruid/python-wayback-machine-downloader
 Author: bitdruid
@@ -99,12 +99,22 @@ Disables following redirects of snapshots. Useful for preventing timestamp-folde
   
 - **`--verbosity`** `<level>`:<br>
 Sets verbosity level. Options are `json` (prints JSON response) or `progress` (shows progress bar).
+<!-- Alternatively set verbosity level to `trace` for a very detailed output. -->
+
+- **`--log`** `<path>`:<br>
+Path defaults to output-dir. Saves a log file with the output of the tool. Named as `waybackup_<sanitized_url>.log`.
+
+- **`--workers`** `<count>`:<br>
+Sets the number of simultaneous download workers. Default is 1, safe range is about 10. Be cautious as too many workers may lead to refused connections from the Wayback Machine.
   
 - **`--retry`** `<attempts>`:<br>
 Specifies number of retry attempts for failed downloads.
-  
-- **`--workers`** `<count>`:<br>
-Sets the number of simultaneous download workers. Default is 1, safe range is about 10. Be cautious as too many workers may lead to refused connections from the Wayback Machine.
+
+- **`--delay`** `<seconds>`:<br>
+Specifies delay between download requests in seconds. Default is no delay (0).
+
+<!-- - **`--convert-links`**:<br>
+If set, all links in the downloaded files will be converted to local links. This is useful for offline browsing. The links are converted to the local path structure. Show output with `--verbosity trace`. -->
 
 **CDX Query Handling:**
 - **`--cdxbackup`** `<path>`:<br>

{pywaybackup-1.3.2 → pywaybackup-1.4.0}/README.md

@@ -83,12 +83,22 @@ Disables following redirects of snapshots. Useful for preventing timestamp-folde
   
 - **`--verbosity`** `<level>`:<br>
 Sets verbosity level. Options are `json` (prints JSON response) or `progress` (shows progress bar).
+<!-- Alternatively set verbosity level to `trace` for a very detailed output. -->
+
+- **`--log`** `<path>`:<br>
+Path defaults to output-dir. Saves a log file with the output of the tool. Named as `waybackup_<sanitized_url>.log`.
+
+- **`--workers`** `<count>`:<br>
+Sets the number of simultaneous download workers. Default is 1, safe range is about 10. Be cautious as too many workers may lead to refused connections from the Wayback Machine.
   
 - **`--retry`** `<attempts>`:<br>
 Specifies number of retry attempts for failed downloads.
-  
-- **`--workers`** `<count>`:<br>
-Sets the number of simultaneous download workers. Default is 1, safe range is about 10. Be cautious as too many workers may lead to refused connections from the Wayback Machine.
+
+- **`--delay`** `<seconds>`:<br>
+Specifies delay between download requests in seconds. Default is no delay (0).
+
+<!-- - **`--convert-links`**:<br>
+If set, all links in the downloaded files will be converted to local links. This is useful for offline browsing. The links are converted to the local path structure. Show output with `--verbosity trace`. -->
 
 **CDX Query Handling:**
 - **`--cdxbackup`** `<path>`:<br>

pywaybackup-1.4.0/pywaybackup/Arguments.py

@@ -0,0 +1,102 @@
+
+import sys
+import os
+import argparse
+
+from pywaybackup.helper import url_split, sanitize_filename
+
+from pywaybackup.__version__ import __version__
+
+class Arguments:
+
+    def __init__(self):
+
+        parser = argparse.ArgumentParser(description='Download from wayback machine (archive.org)')
+        parser.add_argument('-a', '--about', action='version', version='%(prog)s ' + __version__ + ' by @bitdruid -> https://github.com/bitdruid')
+        parser.add_argument('-d', '--debug', action='store_true', help='Debug mode (Always full traceback and creates an error.log')
+
+        required = parser.add_argument_group('required (one exclusive)')
+        required.add_argument('-u', '--url', type=str, metavar="", help='url (with subdir/subdomain) to download')
+        exclusive_required = required.add_mutually_exclusive_group(required=True)
+        exclusive_required.add_argument('-c', '--current', action='store_true', help='download the latest version of each file snapshot')
+        exclusive_required.add_argument('-f', '--full', action='store_true', help='download snapshots of all timestamps')
+        exclusive_required.add_argument('-s', '--save', action='store_true', help='save a page to the wayback machine')
+
+        optional = parser.add_argument_group('optional query parameters')
+        optional.add_argument('-l', '--list', action='store_true', help='only print snapshots (opt range in y)')
+        optional.add_argument('-e', '--explicit', action='store_true', help='search only for the explicit given url')
+        optional.add_argument('-o', '--output', type=str, metavar="", help='output folder - defaults to current directory')
+        optional.add_argument('-r', '--range', type=int, metavar="", help='range in years to search')
+        optional.add_argument('--start', type=int, metavar="", help='start timestamp format: YYYYMMDDhhmmss')
+        optional.add_argument('--end', type=int, metavar="", help='end timestamp format: YYYYMMDDhhmmss')
+
+        special = parser.add_argument_group('manipulate behavior')
+        special.add_argument('--csv', type=str, nargs='?', const=True, metavar='path', help='save a csv file with the json output - defaults to output folder')
+        special.add_argument('--skip', type=str, nargs='?', const=True, metavar='path', help='skips existing files in the output folder by checking the .csv file - defaults to output folder')
+        special.add_argument('--no-redirect', action='store_true', help='do not follow redirects by archive.org')
+        special.add_argument('--verbosity', type=str, default="info", metavar="", help='["progress", "json"] for different output or ["trace"] for very detailed output')
+        special.add_argument('--log', type=str, nargs='?', const=True, metavar='path', help='save a log file - defaults to output folder')
+        special.add_argument('--retry', type=int, default=0, metavar="", help='retry failed downloads (opt tries as int, else infinite)')
+        special.add_argument('--workers', type=int, default=1, metavar="", help='number of workers (simultaneous downloads)')
+        # special.add_argument('--convert-links', action='store_true', help='Convert all links in the files to local paths. Requires -c/--current')
+        special.add_argument('--delay', type=int, default=0, metavar="", help='delay between each download in seconds')
+
+        cdx = parser.add_argument_group('cdx (one exclusive)')
+        exclusive_cdx = cdx.add_mutually_exclusive_group()
+        exclusive_cdx.add_argument('--cdxbackup', type=str, nargs='?', const=True, metavar='path', help='Save the cdx query-result to a file for recurent use - defaults to output folder')
+        exclusive_cdx.add_argument('--cdxinject', type=str, nargs='?', const=True, metavar='path', help='Inject a cdx backup-file to download according to the given url')
+
+        auto = parser.add_argument_group('auto')
+        auto.add_argument('--auto', action='store_true', help='includes automatic csv, skip and cdxbackup/cdxinject to resume a stopped download')
+
+        args = parser.parse_args(args=None if sys.argv[1:] else ['--help']) # if no arguments are given, print help
+
+        # if args.convert_links and not args.current:
+        #     parser.error("--convert-links can only be used with the -c/--current option")
+
+        self.args = args
+
+    def get_args(self):
+        return self.args
+    
+class Configuration:
+
+    @classmethod
+    def init(cls):
+
+        cls.args = Arguments().get_args()
+        for key, value in vars(cls.args).items():
+            setattr(Configuration, key, value)
+
+        # args now attributes of Configuration // Configuration.output, ...
+        cls.command = ' '.join(sys.argv[1:])
+        cls.domain, cls.subdir, cls.filename = url_split(cls.url)
+
+        if cls.output is None:
+            cls.output = os.path.join(os.getcwd(), "waybackup_snapshots")
+        os.makedirs(cls.output, exist_ok=True)
+
+        if cls.log is True:
+            cls.log = os.path.join(cls.output, f"waybackup_{sanitize_filename(cls.url)}.log")
+
+        if cls.full:
+            cls.mode = "full"
+        if cls.current:
+            cls.mode = "current"
+
+        if cls.auto:
+            cls.skip = cls.output
+            cls.csv = cls.output
+            cls.cdxbackup = cls.output
+            cls.cdxinject = os.path.join(cls.output, f"waybackup_{sanitize_filename(cls.url)}.cdx")
+        else:
+            if cls.skip is True:
+                cls.skip = cls.output
+            if cls.csv is True:
+                cls.csv = cls.output
+            if cls.cdxbackup is True:
+                cls.cdxbackup = cls.output
+            if cls.cdxinject is True:
+                cls.cdxinject = cls.output
+
+

pywaybackup-1.4.0/pywaybackup/Converter.py

@@ -0,0 +1,182 @@
+import os
+import errno
+import magic
+from pywaybackup.helper import url_split
+
+from pywaybackup.Arguments import Configuration as config
+from pywaybackup.Verbosity import Verbosity as vb
+import re
+
+class Converter:
+
+    @classmethod
+    def define_root_steps(cls, filepath) -> str:
+        """
+        Define the steps (../) to the root directory.
+        """
+        abs_path = os.path.abspath(filepath)
+        webroot_path = os.path.abspath(f"{config.output}/{config.domain}/") # webroot is the domain folder in the output
+        # common path between the two
+        common_path = os.path.commonpath([abs_path, webroot_path])
+        # steps up to the common path
+        rel_path_from_common = os.path.relpath(abs_path, common_path)
+        steps_up = rel_path_from_common.count(os.path.sep)
+        if steps_up <= 1: # if the file is in the root of the domain
+            return "./"
+        return "../" * steps_up
+
+
+
+
+
+    @classmethod
+    def links(cls, filepath, status_message=None):
+        """
+        Convert all links in a HTML / CSS / JS file to local paths.
+        """
+
+
+        def extract_urls(content) -> list:
+            """
+            Extract all links from a file.
+            """
+
+            #content = re.sub(r'\s+', '', content)
+            #content = re.sub(r'\n', '', content)
+
+            html_types = ["src", "href", "poster", "data-src"]
+            css_types = ["url"]
+            links = []
+            for html_type in html_types:
+                # possible formatings of the value: "url", 'url', url
+                matches = re.findall(f"{html_type}=[\"']?([^\"'>]+)", content)
+                links += matches
+            for css_type in css_types:
+                # possible formatings of the value: url(url) url('url') url("url") // ends with )
+                matches = re.findall(rf"{css_type}\((['\"]?)([^'\"\)]+)\1\)", content)
+                links += [match[1] for match in matches]
+            links = list(set(links))
+            return links
+
+
+        def local_url(original_url, domain, count) -> str:
+            """
+            Convert a given url to a local path.
+            """
+            original_url_domain = url_split(original_url)[0]
+
+            # check if the url is external or internal (external is returned as is because no need to convert)
+            external = False
+            if original_url_domain != domain:
+                if "://" in original_url:
+                    external = True
+            if original_url.startswith("//"):
+                external = True
+            if external:
+                status_message.trace(status="", type=f"{count}/{len(links)}", message="External url")
+                return original_url
+
+            # convert the url to a relative path to the local root (download dir) if it's a valid path, else return the original url
+            original_url_file = os.path.join(config.output, config.domain, normalize_url(original_url))
+            if validate_path(original_url_file):
+                if original_url.startswith("/"): # if only starts with /
+                    original_url = f"{cls.define_root_steps(filepath)}{original_url.lstrip('/')}"
+                if original_url.startswith(".//"):
+                    original_url = f"{cls.define_root_steps(filepath)}{original_url.lstrip('./')}"
+                if original_url_domain == domain: # if url is like https://domain.com/path/to/file
+                    original_url = f"{cls.define_root_steps(filepath)}{original_url.split(domain)[1].lstrip('/')}"
+                if original_url.startswith("../"): # if file is already ../ check if it's not too many steps up
+                    original_url = f"{cls.define_root_steps(filepath)}{original_url.split('../')[-1].lstrip('/')}"
+            else:
+                status_message.trace(status="", type="", message=f"{count}/{len(links)}: URL is not a valid path")
+
+            return original_url
+
+
+
+        
+
+        def normalize_url(url) -> str:
+            """
+            Normalize a given url by removing it's protocol, domain and parent directorie references.
+
+            Example1:
+                - Example input: https://domain.com/path/to/file
+                - Example output: /path/to/file
+
+            Example2
+                - input: ../path/to/file
+                - output: /path/to/file
+            """
+            try:
+                url = "/" + url.split("../")[-1]
+            except IndexError:
+                pass
+            if url.startswith("//"):
+                url = "/" + url.split("//")[1]
+            parsed_url = url_split(url)
+            return f"{parsed_url[1]}/{parsed_url[2]}"
+
+
+        def is_pathname_valid(pathname: str) -> bool:
+            """
+            Check if a given pathname is valid.
+            """
+            if not isinstance(pathname, str) or not pathname:
+                return False
+
+            try:
+                os.lstat(pathname)
+            except OSError as exc:
+                if exc.errno == errno.ENOENT:
+                    return True
+                elif exc.errno in {errno.ENAMETOOLONG, errno.ERANGE}:
+                    return False
+            return True
+
+        def is_path_creatable(pathname: str) -> bool:
+            """
+            Check if a given path is creatable.
+            """
+            dirname = os.path.dirname(pathname) or os.getcwd()
+            return os.access(dirname, os.W_OK)
+
+        def is_path_exists_or_creatable(pathname: str) -> bool:
+            """
+            Check if a given path exists or is creatable.
+            """
+            return is_pathname_valid(pathname) or is_path_creatable(pathname)
+
+        def validate_path(filepath: str) -> bool:
+            """
+            Validate if a given path can exist.
+            """
+            return is_path_exists_or_creatable(filepath)
+
+
+
+
+
+        if os.path.isfile(filepath):
+            if magic.from_file(filepath, mime=True).split("/")[1] == "javascript":
+                status_message.trace(status="Error", type="", message="JS-file is not supported")
+                return
+            try:
+                with open(filepath, "r") as file:
+                    domain = config.domain
+                    content = file.read()
+                    links = extract_urls(content)
+                    status_message.store(message=f"\n-----> Convert: [{len(links)}] links in file")
+                    count = 1
+                    for original_link in links:
+                        status_message.trace(status="ORIG", type=f"{count}/{len(links)}", message=original_link)
+                        new_link = local_url(original_link, domain, count)
+                        if new_link != original_link:
+                            status_message.trace(status="CONV", type=f"{count}/{len(links)}", message=new_link)
+                        content = content.replace(original_link, new_link)
+                        count += 1
+                    file = open(filepath, "w")
+                    file.write(content)
+                    file.close()
+            except UnicodeDecodeError:
+                status_message.trace(status="Error", type="", message="Could not decode file to convert links")

{pywaybackup-1.3.2 → pywaybackup-1.4.0}/pywaybackup/SnapshotCollection.py

@@ -70,10 +70,10 @@ class SnapshotCollection:
             download_dir = os.path.join(output, domain, timestamp, subdir)
         download_file = os.path.abspath(os.path.join(download_dir, filename))
         return download_file
-
+    
 
     @classmethod
-    def snapshot_entry_modify(cls, collection_entry: dict, key: str, value: str):
+    def entry_modify(cls, collection_entry: dict, key: str, value: str):
         """
         Modify a key-value pair in a snapshot entry of the collection (dict).
 

pywaybackup-1.4.0/pywaybackup/Verbosity.py

@@ -0,0 +1,114 @@
+import tqdm
+import json
+from pywaybackup.SnapshotCollection import SnapshotCollection as sc
+
+class Verbosity:
+
+    LEVELS = ["trace", "info"]
+    level = None
+
+    mode = None
+    args = None
+    pbar = None
+
+    log = None
+
+    @classmethod
+    def init(cls, v_args: list, log=None):
+        cls.args = v_args
+        cls.log = open(log, "w") if log else None
+        if cls.args == "progress":
+            cls.mode = "progress"
+        elif cls.args == "json":
+            cls.mode = "json"
+        cls.level = cls.args if cls.args in cls.LEVELS else "info"
+
+    @classmethod
+    def fini(cls):
+        if cls.mode == "progress":
+            if cls.pbar is not None:
+                cls.pbar.close()
+        if cls.mode == "json":
+            print(json.dumps(sc.SNAPSHOT_COLLECTION, indent=4, sort_keys=True))
+        if cls.log:
+            cls.log.close()
+
+    @classmethod
+    def write(cls, status="", type="", message=""):
+        """
+        Write a log line based on the provided status, type, and message.
+        
+        Args:
+            status (str): The status of the log line. (e.g. "SUCCESS", "REDIRECT")
+            type (str): The type of the log line. (e.g. "URL", "FILE")
+            message (str): The message to be logged. (e.g. actual url, file path)
+        """
+        logline = cls.generate_logline(status=status, type=type, message=message)
+        if cls.mode != "progress" and cls.mode != "json":
+            if logline:
+                print(logline)
+        if cls.log:
+            cls.log.write(logline + "\n")
+            cls.log.flush()
+
+    @classmethod
+    def progress(cls, progress: int):
+        if cls.mode == "progress":
+            if cls.pbar is None and progress == 0:
+                maxval = sc.count(collection=True)
+                cls.pbar = tqdm.tqdm(total=maxval, desc="Downloading", unit=" snapshot", ascii="░▒█")
+            if cls.pbar is not None and progress is not None and progress > 0:
+                cls.pbar.update(progress)
+                cls.pbar.refresh()
+
+    @classmethod
+    def generate_logline(cls, status: str = "", type: str = "", message: str = ""):
+
+        if not status and not type:
+            return message
+
+        status_length = 11
+        type_length = 5
+
+        status = status.ljust(status_length)
+        type = type.ljust(type_length)
+
+        log_entry = f"{status} -> {type}: {message}"
+
+        return log_entry
+
+class Message(Verbosity):
+    """
+    Message class representing a message-buffer for the Verbosity class.
+
+    If a message should be stored and stacked for later output.
+    """
+
+    def __init__(self):
+        self.message = {}
+
+    def __str__(self):
+        return self.message
+
+    def store(self, status: str = "", type: str = "", message: str = "", level: str = "info"):
+        if level not in self.message:
+            self.message[level] = []
+        self.message[level].append(super().generate_logline(status, type, message))
+
+    def clear(self):
+        self.message = {}
+
+    def write(self):
+        for level in self.message:
+            if self.check_level(level):
+                for message in self.message[level]:
+                    super().write(message=message)
+        self.clear()
+            
+    def check_level(self, level: str):
+        return super().LEVELS.index(level) >= super().LEVELS.index(self.level)
+
+    def trace(self, status: str = "", type: str = "", message: str = ""):
+        self.store(status, type, message, "trace")
+
+        

pywaybackup-1.4.0/pywaybackup/__version__.py

@@ -0,0 +1 @@
+__version__ = "1.4.0"