sedd 2.2.2__tar.gz → 2.4.0__tar.gz

{sedd-2.2.2/sedd.egg-info → sedd-2.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sedd
-Version: 2.2.2
+Version: 2.4.0
 Summary: Unofficial, community-made tool for downloading the Stack Exchange data dumps
 Author-email: LunarWatcher <oliviawolfie@pm.me>
 License-Expression: MIT

{sedd-2.2.2 → sedd-2.4.0}/README.md

@@ -169,6 +169,7 @@ Exractor CLI supports the following configuration options:
 | `-v` | - | Optional | `false` | Whether or not to enable verbose logging. You do not want to enable this unless you're diagnosing a problem with sedd - the verbose logging includes low-level Selenium output. |
 | - | `--detect <last upload date>` | Optional | - | Enables a mode where sedd checks for the upload timestamp to change, and only then begins the download. This has several caveats. Please read [docs/Automated downloads](docs/Automated downloads.md) for more information before using this feature. |
 | `-u` | `--unsupervised` | Optional | false | Rather than notify you about failures and asking you to resolve them, hard failures that would require manual supervision throw an exception instead. |
+| `-N` | `--no-wipe-part-files` | Optional | false | Don't wipe `.part` files in the download directory on boot. Only ever useful if you're doing cursed shit. Disables automatic download error recovery, and may result in the download completion watcher getting stuck |
 
 #### Captchas and other misc. barriers
 
@@ -299,7 +300,9 @@ cmake --build . -j 8 --config Release
 .\sedd-transformer.exe -i ..\..\downloads -t [formatter type]
 ```
 
-Pass `--help` to see the available formatters for your current version of the data dump transformer.
+Pass `--help` to see the available formatters for your current version of the data dump transformer. There's also some filters available for use, that change the data.
+
+If you only want to filter the data, but keep the XML format, you can use `-t xml` to convert back to XML.
 
 ### Supported transformers
 

{sedd-2.2.2 → sedd-2.4.0}/sedd/cli.py

@@ -14,6 +14,8 @@ class SEDDCLIArgs(argparse.Namespace):
     detect: str | None
     unsupervised: bool
 
+    wipe_part_files: bool
+
 
 parser = argparse.ArgumentParser(
     prog="sedd",
@@ -78,6 +80,22 @@ parser.add_argument(
     )
 )
 
+parser.add_argument(
+    "-N,--no-wipe-part-files",
+    required=False,
+    default=True,
+    action="store_false",
+    dest="wipe_part_files",
+    help=(
+        "When supplied, .part files in the folder won't be wiped. "
+        "By default, wiping part files is enabled, as leaving it interferes "
+        "with a download error recovery mechanism. Supplying this flag "
+        "disables download error recovery, as the system has no way to "
+        "differentiate between a .part file that was there before the "
+        "download started"
+    )
+)
+
 parser.add_argument(
     "--detect",
     required=False,

{sedd-2.2.2 → sedd-2.4.0}/sedd/main.py

@@ -2,11 +2,12 @@ from selenium.webdriver.common.by import By
 from selenium.webdriver.firefox.webdriver import WebDriver
 from selenium.common.exceptions import NoSuchElementException
 from typing import Dict
+import glob
 
-
-from time import sleep
+from time import sleep, monotonic
 
 import re
+import os
 import sys
 import random
 from traceback import print_exception
@@ -17,6 +18,8 @@ from .config import load_sedd_config
 from .data import sites
 from .meta import notifications
 from .watcher.observer import register_pending_downloads_observer
+from .watcher.state import DownloadState
+from .watcher.recovery import LastState
 from . import utils
 
 from .driver import init_output_dir, init_firefox_driver
@@ -190,7 +193,7 @@ def login_or_create(browser: WebDriver, site: str):
             break
 
 
-def download_data_dump(browser: WebDriver, site: str, meta_url: str, etags: Dict[str, str]):
+def download_data_dump(browser: WebDriver, site: str, meta_url: str | None, etags: Dict[str, str]):
     logger.info(f"Downloading data dump from {site}")
 
     def _exec_download(browser: WebDriver):
@@ -257,7 +260,7 @@ def download_data_dump(browser: WebDriver, site: str, meta_url: str, etags: Dict
 
         if args.skip_loaded and meta_loaded:
             pass
-        else:
+        elif meta_url is not None:
             browser.get(f"{meta_url}/users/data-dump-access/current")
             check_cloudflare_intercept(browser)
 
@@ -345,7 +348,134 @@ def await_date_change(args: SEDDCLIArgs, driver):
                 "Please open an issue: https://github.com/LunarWatcher/se-data-dump-transformer/"
             )
             raise
-        
+
+def do_download(site: str):
+    if site not in ["https://meta.stackexchange.com", "https://stackapps.com"]:
+        # https://regex101.com/r/kG6nTN/1
+        meta_url = re.sub(
+            r"(https://(?:[^.]+\.(?=stackexchange))?)", r"\1meta.", site)
+    else:
+        meta_url = None
+
+    main_loaded = utils.is_file_downloaded(args.output_dir, site)
+    meta_loaded = utils.is_file_downloaded(args.output_dir, meta_url) or \
+        meta_url is None
+
+    if args.skip_loaded and main_loaded and meta_loaded:
+        pass
+    else:
+        logger.info(f"Extracting from {site}...")
+
+        login_or_create(browser, site)
+        download_data_dump(
+            browser,
+            site,
+            meta_url,
+            etags
+        )
+
+def clear_part_files():
+    files = glob.glob(
+        os.path.join(
+            args.output_dir,
+            "*.part"
+        )
+    )
+
+    for file in files:
+        logger.debug(
+            "Found existing .part file ({}). Removing...",
+            file
+        )
+        os.remove(file)
+
+def normalize_meta(url: str):
+    if (url.startswith("meta.") and url != "meta.stackexchange.com"):
+        return url.replace("meta.", "")
+    return url
+
+def try_recover_fucked_download(
+        last_sizes: dict[str, LastState],
+        state: DownloadState
+):
+    # TODO: This is a very naive approach. Can we get the information directly
+    # form the webdriver?
+    for path in state.pending:
+        # The path is in the form of the full site URL with the .zip
+        # The .part file is in the form `[first path component].[garbage].[rest]`
+        # meta.stackexchange.com.7z would be 
+        # meta.[garbage.]stackexchange.7z.part
+        spl = path.split(".", 1)
+        matches = glob.glob(
+            os.path.join(
+                args.output_dir,
+                '.*.'.join(spl) + ".part"
+            )
+        )
+
+        if len(matches) == 0:
+            logger.warning("{} has 0 part files. Race condition?", path)
+        elif len(matches) > 1:
+            logger.error(
+                "{} has more than 1 part file! Error recovery impossible",
+                path
+            )
+        else:
+            part_path = matches[0]
+            size = os.path.getsize(part_path)
+
+            last_size = last_sizes.get(
+                path,
+                LastState(monotonic(), 0)
+            )
+
+            # We only commit the size if it's changing
+            if (last_size.last_observed_size < size):
+                last_size.last_observed_size = size
+                last_size.last_observed_change = monotonic()
+
+                last_sizes[path] = last_size
+            else:
+                # If it isn't changing, we allow for a 60 second window 
+                # After 30 seconds, a warning is issued. This is likely useless
+                # in practice, because a failure this long almost certainly
+                # means the download has died
+                # After 60 seconds, we treat the download as failed and move on
+                # with our lives. Nuke the part file, redo the download
+                delta = monotonic() - last_size.last_observed_change
+
+                if (delta > 60):
+                    logger.error(
+                        "{} has failed. {} has been pinned for 60 seconds. "
+                        "Retrying",
+                        path,
+                        part_path
+                    )
+                    del last_sizes[path]
+
+                    os.remove(part_path)
+                    # TODO: optimally, we'd just call a single function that
+                    # directly downloads the specific site. Unfortunately, the
+                    # system wasn't set up to deal with this, and I don't feel
+                    # like rewriting it when all I want is to download the god
+                    # damn file
+                    # The download system should be split up to allow for this,
+                    # but it'll be a bigger refactor to do it. The current URL
+                    # system is fairly fragile, really
+                    do_download(
+                        "https://" + normalize_meta(
+                            path.replace(".7z", "")
+                        )
+                    )
+                elif (
+                    # the "and" is to avoid excessive spam. This allows a 2
+                    # second window
+                    delta > 30 and delta < 32
+                ):
+                    logger.warning(
+                        "No observed change in {} for 30 seconds",
+                        path
+                    )
 
 etags: Dict[str, str] = {}
 
@@ -353,34 +483,19 @@ try:
     state, observer = register_pending_downloads_observer(args.output_dir)
     if args.detect is not None:
         await_date_change(args, browser)
-
+    if args.wipe_part_files:
+        clear_part_files()
     for site in sites.sites:
-        if site not in ["https://meta.stackexchange.com", "https://stackapps.com"]:
-            # https://regex101.com/r/kG6nTN/1
-            meta_url = re.sub(
-                r"(https://(?:[^.]+\.(?=stackexchange))?)", r"\1meta.", site)
-
-        main_loaded = utils.is_file_downloaded(args.output_dir, site)
-        meta_loaded = utils.is_file_downloaded(args.output_dir, meta_url)
-
-        if args.skip_loaded and main_loaded and meta_loaded:
-            pass
-        else:
-            logger.info(f"Extracting from {site}...")
-
-            login_or_create(browser, site)
-            download_data_dump(
-                browser,
-                site,
-                meta_url,
-                etags
-            )
+        do_download(site)
 
     if observer:
         pending = state.size()
 
         logger.info(f"Waiting for {pending} download{'s'[:pending^1]} to complete")
+        if args.wipe_part_files:
+            logger.info("Download error recovery is enabled.")
 
+        last_sizes = {}
         while True:
             if state.empty():
                 observer.stop()
@@ -389,6 +504,11 @@ try:
                 utils.cleanup_archive(args.output_dir)
                 break
             else:
+                if args.wipe_part_files:
+                    try_recover_fucked_download(
+                        last_sizes,
+                        state
+                    )
                 sleep(1)
 
     notifications.notify(

{sedd-2.2.2 → sedd-2.4.0}/sedd/ubo/ubo.py

@@ -23,7 +23,7 @@ def init_ubo_settings(browser: Firefox, config: SEDDConfig, ubo_id: str) -> bool
         ubo_set_advanced_settings(browser, settings)
 
         # idk why, but applyFilterListSelection only works after a delay
-        sleep(1)
+        sleep(3)
 
         ubo_set_selected_filters(browser, settings)
         ubo_set_whitelist(browser, settings)

{sedd-2.2.2 → sedd-2.4.0}/sedd/utils.py

@@ -82,6 +82,8 @@ def cleanup_archive(base_path: str) -> None:
         traceback.print_exception(sys.exception())
 
 
-def is_file_downloaded(base_path: str, site_or_url: str) -> bool:
+def is_file_downloaded(base_path: str, site_or_url: str | None) -> bool:
+    if (site_or_url is None):
+        return False
     file_name = get_file_name(site_or_url)
     return check_file(base_path, file_name)

sedd-2.4.0/sedd/watcher/recovery.py

@@ -0,0 +1,6 @@
+from dataclasses import dataclass
+
+@dataclass
+class LastState:
+    last_observed_change: float
+    last_observed_size: int

{sedd-2.2.2 → sedd-2.4.0/sedd.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sedd
-Version: 2.2.2
+Version: 2.4.0
 Summary: Unofficial, community-made tool for downloading the Stack Exchange data dumps
 Author-email: LunarWatcher <oliviawolfie@pm.me>
 License-Expression: MIT

{sedd-2.2.2 → sedd-2.4.0}/sedd.egg-info/SOURCES.txt

@@ -30,4 +30,5 @@ sedd/ubo/utils.py
 sedd/watcher/__init__.py
 sedd/watcher/handler.py
 sedd/watcher/observer.py
+sedd/watcher/recovery.py
 sedd/watcher/state.py