sedd 0.0.0__py3-none-any.whl

sedd/__main__.py

@@ -0,0 +1 @@
+from .main import *

sedd/cli.py

@@ -0,0 +1,69 @@
+import argparse
+
+from os import getcwd
+from os.path import join
+
+
+class SEDDCLIArgs(argparse.Namespace):
+    skip_loaded: bool
+    keep_consent: bool
+    output_dir: str
+    dry_run: bool
+    disable_undetected: bool
+    verbose: bool
+
+
+parser = argparse.ArgumentParser(
+    prog="sedd",
+    description="Automatic (unofficial) SE data dump downloader for the anti-community data dump format",
+)
+
+parser.add_argument(
+    "-s", "--skip-loaded",
+    required=False,
+    default=False,
+    action="store_true",
+    dest="skip_loaded"
+)
+
+parser.add_argument(
+    "-k", "--keep-consent",
+    required=False,
+    dest="keep_consent",
+    action="store_true",
+    default=False
+)
+
+parser.add_argument(
+    "-o", "--outputDir",
+    required=False,
+    dest="output_dir",
+    default=join(getcwd(), "downloads")
+)
+
+parser.add_argument(
+    "-g", "--disable-undetected-geckodriver",
+    required=False,
+    dest="disable_undetected",
+    action="store_true",
+    default=False
+)
+
+parser.add_argument(
+    "--dry-run",
+    required=False,
+    default=False,
+    action="store_true",
+    dest="dry_run"
+)
+parser.add_argument(
+    "-v",
+    required=False,
+    default=False,
+    action="store_true",
+    dest="verbose"
+)
+
+
+def parse_cli_args() -> SEDDCLIArgs:
+    return parser.parse_args()

sedd/driver.py

@@ -0,0 +1,62 @@
+from os import path, makedirs
+from urllib import request
+from json import dumps
+from uuid import uuid4
+import platform
+
+from selenium import webdriver
+from selenium.webdriver.firefox.options import Options
+from undetected_geckodriver import Firefox as UFirefox
+
+from .config import SEDDConfig
+from .ubo import init_ubo_settings
+
+
+def init_output_dir(output_dir: str):
+    if not path.exists(output_dir):
+        makedirs(output_dir)
+
+    print(output_dir)
+
+    return output_dir
+
+
+def init_firefox_driver(config: SEDDConfig, disable_undetected: bool, output_dir: str):
+    options = Options()
+    options.enable_downloads = True
+    options.set_preference("browser.download.folderList", 2)
+    options.set_preference("browser.download.manager.showWhenStarting", False)
+    options.set_preference("browser.download.dir", output_dir)
+    options.set_preference(
+        "browser.helperApps.neverAsk.saveToDisk", "application/x-gzip"
+    )
+
+    # our own uuid for uBO so as we don't need to do the dance of inspecing internals
+    ubo_internal_uuid = f"{uuid4()}"
+
+    options.set_preference("extensions.webextensions.uuids", dumps(
+        {"uBlock0@raymondhill.net": ubo_internal_uuid}))
+
+    is_apple = platform.system() == "Darwin"
+    use_undetected = not disable_undetected and not is_apple
+    if use_undetected:
+        print("Using undetected-geckodriver")
+        browser = UFirefox(options = options)
+    else:
+        print("Warning: using standard geckodriver. Cloudflare may perpetually block you")
+        if is_apple:
+            print("This option is forced on macOS. For undetected_geckodriver, "
+                  "run the downloader in a Linux or Windows environment.")
+        browser = webdriver.Firefox(options=options)
+
+    ubo_download_url = config.get_ubo_download_url()
+
+    if not path.exists("ubo.xpi"):
+        print(f"Downloading uBO from: {ubo_download_url}")
+        request.urlretrieve(ubo_download_url, "ubo.xpi")
+
+    ubo_id = browser.install_addon("ubo.xpi", temporary=True)
+
+    init_ubo_settings(browser, config, ubo_internal_uuid)
+
+    return browser, ubo_id

sedd/main.py

@@ -0,0 +1,263 @@
+from selenium.webdriver.common.by import By
+from selenium.webdriver.firefox.webdriver import WebDriver
+from selenium.common.exceptions import NoSuchElementException
+from typing import Dict
+
+
+from time import sleep
+
+import re
+import sys
+from traceback import print_exception
+
+
+from .cli import parse_cli_args
+from .config import load_sedd_config
+from .data import sites
+from .meta import notifications
+from .watcher.observer import register_pending_downloads_observer
+from . import utils
+
+from .driver import init_output_dir, init_firefox_driver
+import logging
+
+args = parse_cli_args()
+
+if args.verbose:
+    logging.basicConfig(level=logging.DEBUG)
+
+sedd_config = load_sedd_config()
+
+output_dir = init_output_dir(args.output_dir)
+
+browser, ubo_id = init_firefox_driver(
+    sedd_config,
+    args.disable_undetected,
+    output_dir
+)
+
+
+def kill_cookie_shit(browser: WebDriver):
+    sleep(3)
+    browser.execute_script(
+        """let elem = document.getElementById("onetrust-banner-sdk"); if (elem) { elem.parentNode.removeChild(elem); }""")
+    sleep(1)
+
+def check_cloudflare_intercept(browser: WebDriver):
+    if browser.title == "Just a moment...":
+        print("CF verification hit. Trying soft workaround")
+        sleep(15)
+
+        if (browser.title == "Just a moment..."):
+            print("Irrecoverable state suspected; captcha solving likely required")
+            notifications.notify("CloudFlare verification hit; auto-verification failed. Please complete the captcha", sedd_config)
+        else:
+            print("Auto-recovered from CF wall")
+            return
+
+        while browser.title == "Just a moment...":
+            print("Still stuck on CF verification. Waiting for 10 seconds")
+            sleep(10)
+
+def is_logged_in(browser: WebDriver, site: str):
+    url = f"{site}/users/current"
+    browser.get(url)
+    sleep(1)
+    check_cloudflare_intercept(browser)
+
+    return "/users/" in browser.current_url
+
+
+def login_or_create(browser: WebDriver, site: str):
+    if is_logged_in(browser, site):
+        print("Already logged in")
+    else:
+        print("Not logged in and/or not registered. Logging in now")
+        while True:
+            browser.get(f"{site}/users/login")
+            check_cloudflare_intercept(browser)
+
+            if "?newreg" in browser.current_url:
+                print(f"Auto-created {site} without login needed")
+                break
+
+            email_elem = browser.find_element(By.ID, "email")
+            password_elem = browser.find_element(By.ID, "password")
+            email_elem.send_keys(sedd_config.email)
+            password_elem.send_keys(sedd_config.password)
+            retryLogin = False
+
+            curr_url = browser.current_url
+            browser.find_element(By.ID, "submit-button").click()
+
+            try:
+                elem = browser.find_element(By.CSS_SELECTOR, "#login-form > .js-error-message")
+                if elem is not None:
+                    print("Login failed quietly. Retrying")
+                    continue
+            except:
+                # No error element
+                pass
+
+            check_cloudflare_intercept(browser)
+
+            while browser.current_url == curr_url:
+                sleep(3)
+
+
+            captcha_walled = False
+            while "/nocaptcha" in browser.current_url:
+                if not captcha_walled:
+                    captcha_walled = True
+
+                notifications.notify(
+                    "Captcha wall hit during login", sedd_config
+                )
+
+                sleep(10)
+
+            if captcha_walled or retryLogin:
+                continue
+
+            if not is_logged_in(browser, site):
+                raise RuntimeError("Login failed")
+
+            break
+
+
+def download_data_dump(browser: WebDriver, site: str, meta_url: str, etags: Dict[str, str]):
+    print(f"Downloading data dump from {site}")
+
+    def _exec_download(browser: WebDriver):
+        if args.keep_consent:
+            print('Consent dialog will not be auto-removed')
+        else:
+            kill_cookie_shit(browser)
+
+        try:
+            checkbox = browser.find_element(By.ID, "datadump-agree-checkbox")
+            btn = browser.find_element(By.ID, "datadump-download-button")
+        except NoSuchElementException:
+            raise RuntimeError(f"Bad site: {site}")
+
+        if args.dry_run:
+            return
+
+        browser.execute_script("""
+        (function() {
+            let oldFetch = window.fetch;
+            window.fetch = (url, opts) => {
+                let promise = oldFetch(url, opts);
+
+                if (url.includes("/link")) {
+                    promise.then(res => {
+                        res.clone().json().then(json => {
+                            window.extractedUrl = json["url"];
+                            console.log(extractedUrl);
+                        });
+                        return res;
+                    });
+                    return new Promise(resolve => setTimeout(resolve, 4000))
+                        .then(_ => promise);
+                }
+                return promise;
+            };
+        })();
+        """)
+
+        checkbox.click()
+        sleep(1)
+        btn.click()
+        check_cloudflare_intercept(browser)
+        sleep(2)
+        url = browser.execute_script("return window.extractedUrl;")
+        utils.extract_etag(url, etags)
+
+        sleep(5)
+
+    main_loaded = utils.is_file_downloaded(args.output_dir, site)
+    meta_loaded = utils.is_file_downloaded(args.output_dir, meta_url)
+
+    if not args.skip_loaded or not main_loaded or not meta_loaded:
+        if args.skip_loaded and main_loaded:
+            pass
+        else:
+            browser.get(f"{site}/users/data-dump-access/current")
+            check_cloudflare_intercept(browser)
+
+            if not args.dry_run:
+                utils.archive_file(args.output_dir, site)
+
+            _exec_download(browser)
+
+        if args.skip_loaded and meta_loaded:
+            pass
+        else:
+            browser.get(f"{meta_url}/users/data-dump-access/current")
+            check_cloudflare_intercept(browser)
+
+            if not args.dry_run:
+                utils.archive_file(args.output_dir, meta_url)
+
+            _exec_download(browser)
+
+
+etags: Dict[str, str] = {}
+
+try:
+    state, observer = register_pending_downloads_observer(args.output_dir)
+
+    for site in sites.sites:
+        if site not in ["https://meta.stackexchange.com", "https://stackapps.com"]:
+            # https://regex101.com/r/kG6nTN/1
+            meta_url = re.sub(
+                r"(https://(?:[^.]+\.(?=stackexchange))?)", r"\1meta.", site)
+
+        main_loaded = utils.is_file_downloaded(args.output_dir, site)
+        meta_loaded = utils.is_file_downloaded(args.output_dir, meta_url)
+
+        if args.skip_loaded and main_loaded and meta_loaded:
+            pass
+        else:
+            print(f"Extracting from {site}...")
+
+            login_or_create(browser, site)
+            download_data_dump(
+                browser,
+                site,
+                meta_url,
+                etags
+            )
+
+    if observer:
+        pending = state.size()
+
+        print(f"Waiting for {pending} download{'s'[:pending^1]} to complete")
+
+        while True:
+            if state.empty():
+                observer.stop()
+                browser.quit()
+
+                utils.cleanup_archive(args.output_dir)
+                break
+            else:
+                sleep(1)
+
+except KeyboardInterrupt:
+    pass
+
+except:
+    exception = sys.exc_info()
+
+    try:
+        print_exception(exception)
+    except:
+        print(exception)
+
+    browser.quit()
+finally:
+    # TODO: replace with validation once downloading is verified done
+    # (or export for separate, later verification)
+    # Though keeping it here, removing files and re-running downloads feels like a better idea
+    print(etags)

sedd/utils.py

@@ -0,0 +1,85 @@
+from typing import Dict
+import requests as r
+from urllib.parse import urlparse
+import os.path
+import re
+import sys
+
+from .data.files_map import files_map, inverse_files_map
+from .data.sites import sites
+
+
+def extract_etag(url: str, etags: Dict[str, str]):
+    res = r.get(
+        url,
+        stream=True
+    )
+    if res.status_code != 200:
+        raise RuntimeError(f"Panic: failed to get {url}: {res.status_code}")
+
+    etag = res.headers["ETag"]
+    res.close()
+
+    parsed_url = urlparse(url)
+    path = parsed_url.path
+    filename = os.path.basename(path)
+
+    etags[filename] = etag
+
+    print(f"ETag for {filename}: {etag}")
+
+
+def get_file_name(site_or_url: str) -> str:
+    domain = re.sub(r'https://', '', site_or_url)
+
+    try:
+        file_name = files_map[domain]
+        return f'{file_name}.7z'
+    except KeyError:
+        return f'{domain}.7z'
+
+
+def is_dump_file(file_name: str) -> bool:
+    file_name = re.sub(r'\.7z$', '', file_name)
+
+    try:
+        inverse_files_map[file_name]
+    except KeyError:
+        origin = f'https://{file_name}'
+        return origin in sites
+
+    return True
+
+
+def check_file(base_path: str, file_name: str) -> bool:
+    try:
+        res = os.stat(os.path.join(base_path, file_name))
+        return res.st_size > 0
+    except FileNotFoundError:
+        return False
+
+
+def archive_file(base_path: str, site_or_url: str) -> None:
+    try:
+        file_name = get_file_name(site_or_url)
+        file_path = os.path.join(base_path, file_name)
+        os.rename(file_path, f"{file_path}.old")
+    except FileNotFoundError:
+        pass
+
+
+def cleanup_archive(base_path: str) -> None:
+    try:
+        file_entries = os.listdir(base_path)
+
+        for entry in file_entries:
+            if entry.endswith('.old'):
+                entry_path = os.path.join(base_path, entry)
+                os.remove(entry_path)
+    except:
+        print(sys.exc_info())
+
+
+def is_file_downloaded(base_path: str, site_or_url: str) -> bool:
+    file_name = get_file_name(site_or_url)
+    return check_file(base_path, file_name)

sedd-0.0.0.dist-info/METADATA

@@ -0,0 +1,91 @@
+Metadata-Version: 2.4
+Name: sedd
+Version: 0.0.0
+Summary: Unofficial, community-made tool for downloading the Stack Exchange data dumps
+Author-email: LunarWatcher <oliviawolfie@pm.me>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/LunarWatcher/se-data-dump-transformer
+Project-URL: Documentation, https://github.com/LunarWatcher/se-data-dump-transformer
+Project-URL: Repository, https://github.com/LunarWatcher/se-data-dump-transformer.git
+Project-URL: Issues, https://github.com/LunarWatcher/se-data-dump-transformer/issues
+Project-URL: Changelog, https://github.com/LunarWatcher/se-data-dump-transformer/releases
+Keywords: Stack Exchange,data dump,Stack Exchange data dump,downloader
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: System :: Archiving
+Classifier: Topic :: Utilities
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Operating System :: MacOS
+Classifier: Operating System :: Microsoft :: Windows
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: selenium==4.32.0
+Requires-Dist: undetected-geckodriver-lw>=2.1.0
+Requires-Dist: desktop-notifier==5.0.1
+Requires-Dist: watchdog==4.0.2
+Requires-Dist: requests
+Dynamic: license-file
+
+# SE Data Dump Downloader
+
+
+For more comprehensive information, please read the [main README](https://github.com/LunarWatcher/se-data-dump-transformer/tree/master) on GitHub. This README contains an abridged version of the main README specifically aimed at Pypi users. 
+
+For usage problems not listed in this readme, see the main README. If no information exists, please open an issue on GitHub - keeping the tool accessible to everyone is a priority.
+
+---
+
+The SE Data Dump Downloader (abbreviated `sedd`) is a command line Selenium-based utility for downloading the entire Stack Exchange data dump in their new [anti-community format](https://stackoverflow.com/help/data-dumps), since they decided not to bother providing an official "download all" button. It's one of two components that operate on the data dump in the second project, the other being the (non-python-based) SE data dump transformer - a project that converts the data dump from the not-so-useful official `.xml` format to some other formats. The pypi package is exclusively for the downloader, and does not ship with a copy of the transformer. See the main README if you're looking for the transformer.
+
+
+For the pypi version, you can download it with:
+```python3
+pip3 install sedd
+```
+
+Note that there are some additional steps before you can start using it, that are detailed in this README.
+
+## Configuration
+
+`sedd` requires a special `config.json` file in the current working directory. There's a template available [on GitHub](https://github.com/LunarWatcher/se-data-dump-transformer/blob/master/config.example.json).
+
+The only two fields you _need_ to fill out in the template is the email and password fields with credentials for a Stack Exchange account. You need to be logged in to download the data dumps, so the downloader needs the credentials to log in on your behalf. It doesn't matter if you're logged into SE elsewhere, as Selenium automatically creates a blank profile every time it starts, which won't include any cookies from SE, which means login is required.
+
+> [!tip]
+>
+> The downloader can automatically create new accounts in the network for you, if you don't have all 180-whatever accounts on every site in the network already. You can also create these by hand if you prefer for some reason, but you are not required to have all 180+ accounts before using the downloader.
+
+## System requirements and pitfalls
+
+`sedd` is exclusively Firefox-based, due to Chromium completely gutting support for uBlock Origin and custom filters. You need Firefox installed on your system to use `sedd`.
+
+> [!note]
+> On Linux and Windows-based systems, geckodriver is [slightly modified](https://pypi.org/project/undetected-geckodriver-lw/). This is an anti-anti-bot measure meant to prevent Cloudflare loops. If you're on macOS and get sent in a captcha loop, it's recommended you switch to Windows or Linux - a Linux VM is also an option if you have no way out of Apple's closed-down ecosystem.
+
+Note that Ubuntu users, or other people who (for whatever reason) choose to use the Snap version of Firefox, have to jump through some extra hoops. The native version of Firefox is strongly encouraged, but if you run into problems with the snap version of Firefox and can't or won't switch, you need to define `export SE_GECKODRIVER=/snap/bin/geckodriver`. Selenium can and will find the snap version of `geckodriver` on its own, but for reasons I simply don't understand, it will still fail with several arbitrary errors. 
+
+### Cloudflare issues or download issues.
+
+Stack Exchange has configured Cloudflare to be _highly_ aggressive, especially to certain countries. You will almost certainly run into captchas, and the downloader is designed to deal with this. After an initial attempt  to solve the captcha on its own, you'll be notified (provided you don't disable the notification provider in `config.json`) and asked to solve it manually. 
+
+If, at this point, it appears to succeed, but you're redirected back  to a full-screen Cloudflare captcha wall, you've likely run into a Cloudflare loop. See [the main README](https://github.com/LunarWatcher/se-data-dump-transformer/tree/master?tab=readme-ov-file#cloudflare-loops) for further help. If this doesn't help, please open an issue.
+
+If the downloads start fine, but later suddenly fail for no good reason, you're likely running into general download instability. This especially applies to `stackoverflow.com.7z`, as its massive size simply increases the chance you wait for it long enough that it flakes out. See [the main README](https://github.com/LunarWatcher/se-data-dump-transformer/tree/master?tab=readme-ov-file#download-instability-particularly-of-stackoverflowcom7z) for further help.
+
+The "Warnings" section in the README may contain additional information about other failure modes not listed here in the future. 
+
+## Using the downloader
+
+With `./config.json` in the current working directory and Firefox installed, you can now run the downloader with:
+```python3
+sedd 
+```
+
+For command line flags, see `sedd --help`, or [the main readme](https://github.com/LunarWatcher/se-data-dump-transformer/tree/master?tab=readme-ov-file#cli-options).
+

sedd-0.0.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+sedd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sedd/__main__.py,sha256=8D5a8oKgqd6WA1RUkiKCn4l_PVemtyuckxQut0vDHXM,20
+sedd/cli.py,sha256=PRAwTyrY5csFHsPvBgriPxziGAhGqwSj0XGmnm4XRfQ,1280
+sedd/driver.py,sha256=jfxck8Qy8l1JjkvS55hHXTLsX0ouQjGGLZCA7atUWXI,2096
+sedd/main.py,sha256=Dy0jmHOgt-qr759Kq8lnTmNrKdgPP1ONabeLn7mKqCg,7851
+sedd/utils.py,sha256=niSYc5yDmnPWzFsUu_c3UB16zeQVAIjYX9HzrM17Lrg,2055
+sedd-0.0.0.dist-info/licenses/LICENSE,sha256=1jbxcNcOnucv3gZNjgsGmAvleNJnhH5LkVUpYPqYSOA,1487
+sedd-0.0.0.dist-info/METADATA,sha256=toTxrEAmBjabji1AqPKWhxXRj3SbzIKiHGM2D6rkw-0,6724
+sedd-0.0.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sedd-0.0.0.dist-info/top_level.txt,sha256=LW8YTI3wZnjHrWtSdboYe4QHk2pHgkRE22bje4NKxhU,5
+sedd-0.0.0.dist-info/RECORD,,

sedd-0.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

sedd-0.0.0.dist-info/licenses/LICENSE

@@ -0,0 +1,32 @@
+Note that the MIT license only applies to the software. Any downloaded
+or generated data dumps are under various versions of CC-By-SA:
+
+* CC-By-SA 2.5: https://creativecommons.org/licenses/by-sa/2.5/
+* CC-By-SA 3.0: https://creativecommons.org/licenses/by-sa/3.0/
+* CC-By-SA 4.0: https://creativecommons.org/licenses/by-sa/4.0/
+
+For more details, and date ranges for the licenses, see 
+    https://stackoverflow.com/help/licensing
+
+---
+
+Copyright © 2024 Olivia
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and/or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
+OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

sedd-0.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+sedd