redfetch 1.3.0__py3-none-any.whl

redfetch/download.py

@@ -0,0 +1,312 @@
+# standard
+import os
+import shutil
+import time
+import hashlib
+from zipfile import ZipFile, is_zipfile
+import asyncio
+
+# third-party
+import httpx
+import aiofiles
+from tenacity import (
+    retry,
+    stop_after_attempt,
+    wait_exponential,
+    retry_if_exception_type,
+)
+
+# local
+from redfetch import config
+from redfetch.utils import is_safe_path
+
+# ZIP safety constants
+MAX_UNCOMPRESSED_SIZE = 2 * 1024 * 1024 * 1024  # 2GB limit
+MAX_FILES_PER_ZIP = 60000  # Safety cap to avoid zipbombs
+
+
+async def download_install_target_async(
+    *,
+    client: httpx.AsyncClient,
+    resource_id: str,
+    download_url: str,
+    filename: str,
+    file_hash: str | None,
+    folder_path: str,
+    should_flatten: bool = False,
+    protected_files: list[str] | None = None,
+) -> bool:
+    try:
+        print(f"Starting download: {filename} (ID: {resource_id})", flush=True)
+        file_path = os.path.join(folder_path, filename)
+        ok = await download_file_async(client, download_url, file_path, file_hash)
+        if not ok:
+            print(f"Download failed for resource {resource_id}.")
+            return False
+        if is_zipfile(file_path):
+            extracted = await asyncio.to_thread(
+                extract_and_discard_zip,
+                file_path,
+                folder_path,
+                resource_id,
+                should_flatten,
+                protected_files,
+            )
+            if not extracted:
+                print(f"Extraction failed for resource {resource_id}.")
+                return False
+        return True
+    except httpx.HTTPError as e:
+        print(f"Failed to fetch or download resource {resource_id}: {str(e)}")
+        return False
+
+
+@retry(
+    stop=stop_after_attempt(3),
+    wait=wait_exponential(multiplier=0.5, min=0.5, max=4),
+    retry=retry_if_exception_type((httpx.TimeoutException, httpx.NetworkError, httpx.HTTPStatusError)),
+    reraise=True,
+)
+async def download_file_async(
+    client: httpx.AsyncClient,
+    download_url: str,
+    file_path: str,
+    expected_md5: str | None = None,
+) -> bool:
+    """
+    Download a file to disk using an async flow:
+      1. open HTTP stream
+      2. stream body into a temp file with async for / await
+      3. optionally verify MD5
+      4. atomically move temp file into place
+    """
+    os.makedirs(os.path.dirname(file_path), exist_ok=True)
+
+    hasher = hashlib.md5() if expected_md5 else None
+    tmp_path = None
+
+    try:
+        async with client.stream("GET", download_url, timeout=60.0, follow_redirects=True) as resp:
+            resp.raise_for_status()
+            _ensure_disk_space(resp, file_path)
+
+            # Core async loop lives in a small helper
+            tmp_path = await _stream_to_tempfile(resp, file_path, hasher)
+
+        if expected_md5 and hasher:
+            if not _hash_matches(hasher, expected_md5, file_path):
+                return False
+
+        if tmp_path is None:
+            return False
+
+        os.replace(tmp_path, file_path)
+        print(f"Downloaded file {file_path}")
+        return True
+    finally:
+        try:
+            if tmp_path and os.path.exists(tmp_path):
+                os.remove(tmp_path)
+        except Exception:
+            pass
+
+
+def _ensure_disk_space(resp, file_path: str) -> None:
+    """Disk space guard using Content-Length when available."""
+    content_length = resp.headers.get("Content-Length")
+    if content_length is None or not content_length.isdigit():
+        return
+
+    expected_size = int(content_length)
+    target_dir = os.path.dirname(file_path)
+    free_bytes = shutil.disk_usage(target_dir).free
+    # Require a 100MB cushion
+    if free_bytes < expected_size + 100 * 1024 * 1024:
+        raise OSError(f"Insufficient disk space for download: need ~{expected_size} bytes")
+
+
+async def _stream_to_tempfile(resp, file_path: str, hasher) -> str:
+    """Stream HTTP body into a temp file."""
+    tmp_dir = os.path.dirname(file_path)
+    tmp_path = None
+
+    try:
+        async with aiofiles.tempfile.NamedTemporaryFile("wb", delete=False, dir=tmp_dir) as tmp:
+            tmp_path = tmp.name
+
+            async for chunk in resp.aiter_bytes(chunk_size=262_144):
+                if not chunk:
+                    continue
+
+                if hasher:
+                    hasher.update(chunk)
+
+                await tmp.write(chunk)
+
+        return tmp_path
+    except Exception:
+        # Clean up any partially written temp file on error
+        if tmp_path and os.path.exists(tmp_path):
+            os.remove(tmp_path)
+        raise
+
+
+def _hash_matches(hasher, expected_md5: str, file_path: str) -> bool:
+    """Return True when the computed MD5 matches *expected_md5*."""
+    actual = hasher.hexdigest().lower()
+    expected = expected_md5.strip().lower()
+
+    if actual == expected:
+        return True
+
+    print(f"MD5 mismatch for {os.path.basename(file_path)}")
+    print(f"  Expected: {expected}")
+    print(f"  Got:      {actual}")
+    return False
+
+
+#
+# zip functions
+#
+
+def extract_and_discard_zip(zip_path, extract_to, resource_id, should_flatten=False, protected_files=None) -> bool:
+    try:
+        zip_size = os.path.getsize(zip_path)
+        if zip_size > MAX_UNCOMPRESSED_SIZE:
+            print(f"ZIP file {zip_path} exceeds the 2GB size limit. Extraction aborted.")
+            return False
+
+        with ZipFile(zip_path, 'r') as zip_ref:
+            try:
+                bad_member = zip_ref.testzip()
+                if bad_member is not None:
+                    print(f"ZIP integrity check failed at member: {bad_member}. Extraction aborted.")
+                    return False
+            except Exception as e:
+                print(f"ZIP integrity check failed: {e}. Extraction aborted.")
+                return False
+
+            infos = zip_ref.infolist()
+            if len(infos) > MAX_FILES_PER_ZIP:
+                print(f"ZIP has too many files ({len(infos)} > {MAX_FILES_PER_ZIP}). Extraction aborted.")
+                return False
+
+            total_uncompressed_size = sum(zinfo.file_size for zinfo in infos)
+            if total_uncompressed_size > MAX_UNCOMPRESSED_SIZE:
+                print(f"Total uncompressed size {total_uncompressed_size} exceeds the 2GB limit. Extraction aborted.")
+                return False
+
+            try:
+                free_bytes = shutil.disk_usage(extract_to).free
+                if free_bytes < total_uncompressed_size + 500 * 1024 * 1024:
+                    print("Insufficient disk space for extraction. Extraction aborted.")
+                    return False
+            except Exception:
+                pass
+
+            if protected_files is None:
+                protected_files = config.settings.from_env(config.settings.ENV).PROTECTED_FILES_BY_RESOURCE.get(resource_id, [])
+            if should_flatten:
+                extract_flattened(zip_ref, extract_to, protected_files)
+            else:
+                extract_with_structure(zip_ref, extract_to, protected_files)
+            return True
+    finally:
+        delete_zip_file(zip_path)
+
+
+def extract_flattened(zip_ref, extract_to, protected_files):
+    print(f"Flattening extraction to {extract_to}")
+    protected_files_lower = {f.lower() for f in protected_files}
+    for member in zip_ref.infolist():
+        filename = os.path.basename(member.filename)
+        if not filename:
+            continue
+        target_path = os.path.join(extract_to, filename)
+        normalized_path = os.path.normpath(target_path)
+        if is_protected(filename, normalized_path, protected_files, protected_files_lower):
+            print(f"Skipping protected file {filename}")
+            continue
+        if is_safe_path(extract_to, normalized_path):
+            extract_zip_member(zip_ref, member, normalized_path)
+        else:
+            print(f"Skipping unsafe file {member.filename}")
+
+
+def extract_with_structure(zip_ref, extract_to, protected_files):
+    print(f"Extracting with structure to {extract_to}")
+    protected_files_lower = {f.lower() for f in protected_files}
+    for member in zip_ref.infolist():
+        target_path = os.path.join(extract_to, member.filename)
+        normalized_path = os.path.normpath(target_path)
+        if not is_safe_path(extract_to, normalized_path):
+            print(f"Skipping unsafe file {member.filename}")
+            continue
+        if is_protected(os.path.basename(member.filename), normalized_path, protected_files, protected_files_lower):
+            print(f"Skipping protected file {member.filename}")
+            continue
+        if member.is_dir():
+            os.makedirs(normalized_path, exist_ok=True)
+            continue
+        extract_zip_member(zip_ref, member, normalized_path)
+
+
+def extract_zip_member(zip_ref, member, target_path):
+    os.makedirs(os.path.dirname(target_path), exist_ok=True)
+    max_retries = 3
+    retry_delay = 10  # seconds
+
+    for attempt in range(max_retries):
+        try:
+            with zip_ref.open(member) as source, open(target_path, 'wb') as target:
+                shutil.copyfileobj(source, target)
+            return  # Successful extraction, exit the function
+        except PermissionError:
+            file_name = os.path.basename(target_path)
+            folder_path = os.path.dirname(target_path)
+            
+            error_msg = [
+                f"\nPermission Error: Unable to extract {file_name}",
+                "\nThis could be because:",
+                "1. The file is currently in use by another program (e.g., MacroQuest, EQBCS)",
+                "2. You don't have write permissions for this location",
+                "\nPossible solutions:",
+                "1. Close all EverQuest-related programs (MacroQuest, EQBCS, etc.)",
+                f"2. Change the installation directory in settings to a location you own",
+                f"3. Manually set write permissions on: {folder_path}",
+            ]
+            
+            if attempt < max_retries - 1:
+                error_msg.append(f"\nRetrying in {retry_delay} seconds... (Attempt {attempt + 1} of {max_retries})")
+                print("\n".join(error_msg))
+                time.sleep(retry_delay)
+            else:
+                error_msg.append("\nMaximum retry attempts reached. Please resolve the permission issue and try again.")
+                print("\n".join(error_msg))
+                raise PermissionError(f"Failed to extract {file_name} after {max_retries} attempts.")
+        except Exception as e:
+            print(f"Unexpected error while extracting {os.path.basename(target_path)}: {str(e)}")
+            raise
+
+
+def delete_zip_file(zip_path):
+    try:
+        os.remove(zip_path)
+    except PermissionError as e:
+        print(f"PermissionError: Unable to delete zip file {zip_path}. Error: {e}")
+
+
+#
+# utility functions
+#
+
+def is_protected(filename, target_path, protected_files, protected_files_lower):
+    # Overwrite protection for specified files
+    filename_lower = filename.lower()
+    if filename_lower in protected_files_lower and os.path.exists(target_path):
+        # Retrieve the original filename case for message consistency
+        protected_list = [f for f in protected_files if f.lower() == filename_lower]
+        original_filename = protected_list[0] if protected_list else filename
+        print(f"Protected {original_filename}, skipping extraction.")
+        return True
+    return False

redfetch/listener.py

@@ -0,0 +1,216 @@
+"""Local API for the RedGuides.com web interface."""
+
+import asyncio
+import logging
+import webbrowser
+from typing import Any, Dict, Optional
+
+from aiohttp import web
+
+from redfetch import store
+from redfetch import sync
+from redfetch.special import compute_special_status
+
+log = logging.getLogger(__name__)
+
+
+REDGUIDES_ORIGIN = "https://www.redguides.com"
+
+
+@web.middleware
+async def cors_middleware(request: web.Request, handler):
+    """Simple CORS middleware allowing RedGuides origin."""
+    if request.method == "OPTIONS":
+        resp = web.Response(status=204)
+    else:
+        resp = await handler(request)
+
+    resp.headers["Access-Control-Allow-Origin"] = REDGUIDES_ORIGIN
+    resp.headers["Access-Control-Allow-Methods"] = "GET, POST, OPTIONS"
+    resp.headers["Access-Control-Allow-Headers"] = "Content-Type"
+    resp.headers["Access-Control-Allow-Credentials"] = "true"
+    return resp
+
+
+async def _get_root_version_local_async(db_name: str, resource_id: str) -> Optional[int]:
+    """Async helper to fetch root version_local for a tracked root install target."""
+    db_path = store.get_db_path(db_name)
+    try:
+        return await store.fetch_root_version_local(db_path, resource_id)
+    except Exception as e:
+        if "no such table" in str(e):
+            # Table doesn't exist yet; treat as not installed
+            return None
+        log.error("Database error during health check: %s", e, exc_info=True)
+        raise
+
+
+async def handle_health(request: web.Request) -> web.Response:
+    db_name = request.app["db_name"]
+
+    resource_id = request.query.get("resource_id")
+    remote_version_str = request.query.get("remote_version")
+
+    if resource_id and remote_version_str is not None:
+        try:
+            remote_version = int(remote_version_str)
+        except ValueError:
+            return web.json_response({"success": False, "message": "Invalid remote_version"}, status=400)
+
+        try:
+            version_local = await _get_root_version_local_async(db_name, resource_id)
+        except Exception:
+            return web.json_response({"success": False, "message": "Database error"}, status=500)
+
+        if version_local is None:
+            return web.json_response({"action": "install"})
+        elif version_local < remote_version:
+            return web.json_response({"action": "update"})
+        else:
+            return web.json_response({"action": "re-install"})
+
+    return web.json_response({"status": "up"})
+
+
+async def handle_download(request: web.Request) -> web.Response:
+    app = request.app
+    db_name = app["db_name"]
+    headers = app["headers"]
+
+    try:
+        payload: Dict[str, Any] = await request.json()
+    except Exception:
+        return web.json_response({"success": False, "message": "JSON body is required."}, status=400)
+
+    resource_id = payload.get("resource_id")
+    if resource_id is None:
+        return web.json_response({"success": False, "message": "Resource ID is required."}, status=400)
+
+    try:
+        resource_id_str = str(resource_id)
+        db_path = store.get_db_path(db_name)
+        success = await sync.run_sync(db_path, headers, resource_ids=[resource_id_str])
+        if success:
+            return web.json_response({"success": True, "message": "Download completed successfully."})
+        return web.json_response({"success": False, "message": "Download failed due to internal error."}, status=500)
+    except Exception as e:
+        log.error("Error during download: %s", e, exc_info=True)
+        return web.json_response({"success": False, "message": f"Download failed: {e}"}, status=500)
+
+
+async def handle_download_watched(request: web.Request) -> web.Response:
+    app = request.app
+    db_name = app["db_name"]
+    headers = app["headers"]
+
+    try:
+        db_path = store.get_db_path(db_name)
+        success = await sync.run_sync(db_path, headers)
+        if success:
+            return web.json_response(
+                {"success": True, "message": "All watched resources downloaded successfully."}
+            )
+        return web.json_response(
+            {"success": False, "message": "Download of one or more resources failed."}, status=500
+        )
+    except Exception as e:
+        log.error("Error during download of watched resources: %s", e, exc_info=True)
+        return web.json_response({"success": False, "message": f"Download failed: {e}"}, status=500)
+
+
+async def handle_reset_download_date(request: web.Request) -> web.Response:
+    app = request.app
+    db_name = app["db_name"]
+
+    try:
+        payload: Dict[str, Any] = await request.json()
+    except Exception:
+        return web.json_response({"success": False, "message": "JSON body is required."}, status=400)
+
+    resource_id = payload.get("resource_id")
+    if not resource_id:
+        return web.json_response({"success": False, "message": "Resource ID is required."}, status=400)
+
+    try:
+        int(str(resource_id))
+    except ValueError:
+        return web.json_response({"success": False, "message": "Invalid resource ID format."}, status=400)
+
+    def _reset() -> bool:
+        try:
+            with store.get_db_connection(db_name) as conn:
+                cursor = conn.cursor()
+                store.reset_versions_for_resource(cursor, str(resource_id))
+                conn.commit()
+            return True
+        except Exception as e:
+            log.error("Error during resetting download date: %s", e, exc_info=True)
+            return False
+
+    ok = await asyncio.to_thread(_reset)
+    if ok:
+        return web.json_response({"success": True, "message": "Download date reset successfully."})
+    return web.json_response({"success": False, "message": "Reset failed."}, status=500)
+
+
+async def handle_category_map(request: web.Request) -> web.Response:
+    category_map = request.app["category_map"]
+    return web.json_response(list(category_map.keys()))
+
+
+async def handle_special_resource_ids(request: web.Request) -> web.Response:
+    status = compute_special_status(None)
+    special_resource_ids = [int(rid) for rid, info in status.items() if info["is_special"]]
+    return web.json_response(special_resource_ids)
+
+
+async def create_app(
+    settings,
+    db_name: str,
+    headers: dict,
+    category_map,
+) -> web.Application:
+    """Create the aiohttp application for the RedGuides interface."""
+    app = web.Application(middlewares=[cors_middleware])
+
+    app["settings"] = settings
+    app["db_name"] = db_name
+    app["headers"] = headers
+    app["category_map"] = category_map
+
+    app.router.add_get("/health", handle_health)
+    app.router.add_post("/download", handle_download)
+    app.router.add_post("/download-watched", handle_download_watched)
+    app.router.add_post("/reset-download-date", handle_reset_download_date)
+    app.router.add_get("/category-map", handle_category_map)
+    app.router.add_get("/special-resource-ids", handle_special_resource_ids)
+
+    return app
+
+
+async def run_server_async(
+    settings,
+    db_name: str,
+    headers: dict,
+    category_map,
+) -> None:
+    """Run the interface server until cancelled."""
+    app = await create_app(settings, db_name, headers, category_map)
+
+    runner = web.AppRunner(app)
+    await runner.setup()
+    site = web.TCPSite(runner, "127.0.0.1", 7734)
+    await site.start()
+
+    webbrowser.open_new("https://www.redguides.com/cookie/set_marker.php")
+    log.info("Server starting. Browse resources on https://www.redguides.com/community/resources")
+
+    try:
+        # Wait indefinitely until the task is cancelled by the caller (CLI or TUI).
+        await asyncio.Event().wait()
+    except asyncio.CancelledError:
+        log.info("Server task cancelled, shutting down...")
+    finally:
+        await runner.cleanup()
+        log.info("Server stopped.")
+