redfetch 1.3.0__tar.gz → 1.4.0__tar.gz

{redfetch-1.3.0 → redfetch-1.4.0}/.github/workflows/pipeline.yml

@@ -60,6 +60,13 @@ jobs:
         run: |
           hatch build
 
+      - name: Upload Python distribution
+        if: github.ref == 'refs/heads/develop'
+        uses: actions/upload-artifact@v7
+        with:
+          name: python-dist
+          path: dist/
+
       - name: Get Version
         id: get_version
         run: |
@@ -89,22 +96,26 @@ jobs:
         with:
           fetch-depth: 0  # Ensure full Git history is available for versioning
 
+      - name: Download Python distribution
+        if: github.ref == 'refs/heads/develop'
+        uses: actions/download-artifact@v8
+        with:
+          name: python-dist
+          path: dist
+
       - name: Set Environment Variables
         id: set_env_vars
         shell: pwsh
         run: |
-          # Set static environment variables
-          "PYAPP_PROJECT_NAME=redfetch" >> $env:GITHUB_ENV
           "PYAPP_EXEC_MODULE=redfetch.main" >> $env:GITHUB_ENV
           "PYAPP_PASS_LOCATION=1" >> $env:GITHUB_ENV
 
-          # Conditionally set PYAPP_PIP_EXTRA_ARGS based on the branch or tag
           if ($env:GITHUB_REF -eq 'refs/heads/develop') {
-            $pipArgs = "--index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/"
+            "PYAPP_EMBED_WHEEL=1" >> $env:GITHUB_ENV
           } else {
-            $pipArgs = ""
+            "PYAPP_PROJECT_NAME=redfetch" >> $env:GITHUB_ENV
           }
-          "PYAPP_PIP_EXTRA_ARGS=$pipArgs" >> $env:GITHUB_ENV
+          "PYAPP_PIP_EXTRA_ARGS=" >> $env:GITHUB_ENV
 
           # Output the values for verification
           Get-Content $env:GITHUB_ENV
@@ -145,18 +156,23 @@ jobs:
 
       - name: Build PyApp Executable
         run: |
-          $env:PYAPP_PROJECT_NAME = "${{ env.PYAPP_PROJECT_NAME }}"
+          if ($env:PYAPP_EMBED_WHEEL -eq '1') {
+            # Develop embeds the built wheel so its deps resolve from PyPI rather than TestPyPI
+            $wheel = Get-ChildItem -Path "$env:GITHUB_WORKSPACE\dist\*.whl" | Select-Object -First 1
+            if (-not $wheel) { throw "No redfetch wheel found in dist/" }
+            $env:PYAPP_PROJECT_PATH = $wheel.FullName
+            Remove-Item Env:PYAPP_PROJECT_NAME -ErrorAction SilentlyContinue
+            Remove-Item Env:PYAPP_PROJECT_VERSION -ErrorAction SilentlyContinue
+          }
+
           $env:PYAPP_EXEC_MODULE = "${{ env.PYAPP_EXEC_MODULE }}"
           $env:PYAPP_PASS_LOCATION = "${{ env.PYAPP_PASS_LOCATION }}"
           $env:PYAPP_PIP_EXTRA_ARGS = "${{ env.PYAPP_PIP_EXTRA_ARGS }}"
-          $env:PYAPP_PROJECT_VERSION = "${{ env.PYAPP_PROJECT_VERSION }}"
-          
-          Write-Host "PYAPP_PROJECT_NAME: $env:PYAPP_PROJECT_NAME"
+
           Write-Host "PYAPP_EXEC_MODULE: $env:PYAPP_EXEC_MODULE"
           Write-Host "PYAPP_PASS_LOCATION: $env:PYAPP_PASS_LOCATION"
           Write-Host "PYAPP_PIP_EXTRA_ARGS: $env:PYAPP_PIP_EXTRA_ARGS"
-          Write-Host "PYAPP_PROJECT_VERSION: $env:PYAPP_PROJECT_VERSION"
-          
+
           cargo build --release
         working-directory: .\pyapp-latest
         shell: pwsh

{redfetch-1.3.0 → redfetch-1.4.0}/.gitignore

@@ -7,6 +7,10 @@ __pycache__/
 #virtual environments
 env/
 venv/
+.venv/
+
+# uv (project is hatch-managed)
+uv.lock
 
 # Downloaded files
 Downloads/

redfetch-1.4.0/AGENTS.md

@@ -0,0 +1,21 @@
+# AGENTS.md
+
+**redfetch** — a CLI/TUI for downloading and publishing MacroQuest and EverQuest scripts and software via the RedGuides API.
+
+## Running tests
+
+Use `hatch test` — not `pytest`, `uv`, or an ad-hoc venv
+
+```sh
+hatch test                       # full suite
+hatch test tests/test_check.py   # single file
+```
+## Libraries
+
+- [Textual](https://textual.textualize.io/) - TUI
+- [Hatch](https://hatch.pypa.io/latest/) - Build system
+- [PYPA](https://www.pypa.io/en/latest/specifications/) - Package index
+
+## Conventions
+
+- Use the `dev` environment (`hatch shell dev`) for development

{redfetch-1.3.0 → redfetch-1.4.0}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.4
 Name: redfetch
-Version: 1.3.0
+Version: 1.4.0
 Summary: Download and publish EverQuest scripts and software using the RedGuides API
 Project-URL: Homepage, https://www.redguides.com
 Project-URL: Documentation, https://www.redguides.com/community/resources/redfetch.3177/
 Project-URL: Repository, https://github.com/RedGuides/redfetch
 Project-URL: Issues, https://github.com/RedGuides/redfetch/issues
 Project-URL: Changelog, https://github.com/RedGuides/redfetch/blob/main/CHANGELOG.md
-Project-URL: Source_archive, https://github.com/RedGuides/redfetch/archive/aaf2d42ab82d45abeadfafa943e2b61abba2cb60.zip
+Project-URL: Source_archive, https://github.com/RedGuides/redfetch/archive/ccf5cf558b6e6f261fab4079e837d5cb47e7975d.zip
 Author-email: Redbot <ask@redguides.com>
 License-Expression: GPL-3.0-or-later
 License-File: LICENSE
@@ -21,9 +21,10 @@ Requires-Python: >=3.12
 Requires-Dist: aiofiles~=25.1
 Requires-Dist: aiohttp~=3.14
 Requires-Dist: aiosqlite~=0.22
-Requires-Dist: cachetools~=7.1
 Requires-Dist: diskcache~=5.6
-Requires-Dist: dynaconf<3.3,~=3.2
+Requires-Dist: dynaconf>=3.3.2,~=3.3
+Requires-Dist: filelock~=3.29
+Requires-Dist: hishel[async,httpx]~=1.3
 Requires-Dist: httpx[http2]~=0.28
 Requires-Dist: keepachangelog~=2.0
 Requires-Dist: keyring~=25.7
@@ -32,7 +33,6 @@ Requires-Dist: packaging~=26.2
 Requires-Dist: platformdirs~=4.10
 Requires-Dist: psutil~=7.2
 Requires-Dist: pydantic~=2.13
-Requires-Dist: pyperclip~=1.11
 Requires-Dist: pywin32; sys_platform == 'win32'
 Requires-Dist: rich~=15.0
 Requires-Dist: tenacity~=9.1

{redfetch-1.3.0 → redfetch-1.4.0}/pyproject.toml

@@ -21,9 +21,10 @@ dependencies = [
     "aiofiles~=25.1",
     "aiosqlite~=0.22",
     "aiohttp~=3.14",
-    "cachetools~=7.1",
     "diskcache~=5.6",
-    "dynaconf~=3.2,<3.3",  # 3.3 has a bug
+    "dynaconf~=3.3,>=3.3.2",  # (pr 1409 fixed RecursionError)
+    "filelock~=3.29",
+    "hishel[httpx,async]~=1.3",
     "httpx[http2]~=0.28",
     "keepachangelog~=2.0",
     "keyring~=25.7",
@@ -32,7 +33,6 @@ dependencies = [
     "pydantic~=2.13",
     "platformdirs~=4.10",
     "psutil~=7.2",
-    "pyperclip~=1.11",
     "rich~=15.0",
     "tenacity~=9.1",
     "textual~=8.2",

{redfetch-1.3.0 → redfetch-1.4.0}/src/redfetch/__about__.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.3.0'
-__version_tuple__ = version_tuple = (1, 3, 0)
+__version__ = version = '1.4.0'
+__version_tuple__ = version_tuple = (1, 4, 0)
 
 __commit_id__ = commit_id = None

redfetch-1.4.0/src/redfetch/api.py

@@ -0,0 +1,50 @@
+"""Resource API client: fetch_*() takes a client, get_*() is self-contained."""
+
+import httpx
+from redfetch import net
+from redfetch.sync_types import SyncInfo
+
+BASE_URL = net.BASE_URL
+
+
+async def fetch_sync_info(client: httpx.AsyncClient) -> SyncInfo:
+    """Per-user sync info"""
+    url = f'{BASE_URL}/api/rgsync'
+    is_level_2 = is_moderator = False
+    watched: set[str] = set()
+    licenses: list[dict] = []
+    page = 1
+    while True:
+        data = await net.get_json(client, url, params={'page': page})
+        is_level_2 = bool(data.get('is_level_2', False))
+        is_moderator = bool(data.get('is_moderator', False))
+        watched.update(str(rid) for rid in data.get('watched', []))
+        licenses.extend(data.get('licenses', []))
+        if not data.get('pagination', {}).get('has_more', False):
+            break
+        page += 1
+
+    licensed_ids = {
+        str(lic['resource_id']) for lic in licenses if lic.get('resource_id') is not None
+    }
+    return SyncInfo(
+        is_level_2=is_level_2,
+        is_moderator=is_moderator,
+        watched=watched,
+        licensed_ids=licensed_ids,
+        licenses=licenses,
+    )
+
+
+async def get_sync_info(headers: dict) -> SyncInfo:
+    """fetch_sync_info for callers that hold headers but not a client."""
+    async with httpx.AsyncClient(headers=headers, http2=True, timeout=30.0) as client:
+        return await fetch_sync_info(client)
+
+
+async def get_resource_details(resource_id: int, headers: dict) -> dict:
+    """Retrieve details of a specific resource from the API."""
+    url = f'{BASE_URL}/api/resources/{resource_id}'
+    async with httpx.AsyncClient(headers=headers, http2=True, timeout=30.0) as client:
+        data = await net.get_json(client, url)
+    return data['resource']

{redfetch-1.3.0 → redfetch-1.4.0}/src/redfetch/config.py

@@ -8,6 +8,7 @@ import shutil
 # third-party
 import tomlkit
 from dynaconf import Dynaconf, Validator, ValidationError
+from dynaconf.loaders import env_loader
 from platformdirs import user_config_dir, user_data_dir
 
 # Parent Category to folder
@@ -459,25 +460,8 @@ def update_setting(setting_path, setting_value, env=None):
 
 
 def write_env_to_file(new_env):
-    """Update the environment setting in the .env file."""
+    """Persist the selected environment to the .env file; dynaconf reads it back via env_switcher."""
     if env_file_path is None:
         raise RuntimeError("Configuration has not been initialized. Call initialize_config() first.")
 
-    # Read the existing content of the .env file
-    with open(env_file_path, 'r') as file:
-        lines = file.readlines()
-
-    # Update the environment line
-    updated = False
-    for i, line in enumerate(lines):
-        if line.startswith('REDFETCH_ENV='):
-            lines[i] = f'REDFETCH_ENV={new_env}\n'
-            updated = True
-            break
-
-    # If the environment line was not found, add it
-    if not updated:
-        lines.append(f'REDFETCH_ENV={new_env}\n')
-
-    # Write the updated content back to the .env file
-    atomic_write_text(env_file_path, ''.join(lines))
+    env_loader.write(env_file_path, {"REDFETCH_ENV": new_env})

{redfetch-1.3.0 → redfetch-1.4.0}/src/redfetch/download.py

@@ -1,8 +1,12 @@
 # standard
 import os
 import shutil
-import time
 import hashlib
+import stat
+import sys
+import time
+import zlib
+from pathlib import Path
 from zipfile import ZipFile, is_zipfile
 import asyncio
 
@@ -206,6 +210,7 @@ def extract_and_discard_zip(zip_path, extract_to, resource_id, should_flatten=Fa
 
             if protected_files is None:
                 protected_files = config.settings.from_env(config.settings.ENV).PROTECTED_FILES_BY_RESOURCE.get(resource_id, [])
+            sweep_stale_swap_files(extract_to)
             if should_flatten:
                 extract_flattened(zip_ref, extract_to, protected_files)
             else:
@@ -218,6 +223,7 @@ def extract_and_discard_zip(zip_path, extract_to, resource_id, should_flatten=Fa
 def extract_flattened(zip_ref, extract_to, protected_files):
     print(f"Flattening extraction to {extract_to}")
     protected_files_lower = {f.lower() for f in protected_files}
+    pairs = []
     for member in zip_ref.infolist():
         filename = os.path.basename(member.filename)
         if not filename:
@@ -228,14 +234,16 @@ def extract_flattened(zip_ref, extract_to, protected_files):
             print(f"Skipping protected file {filename}")
             continue
         if is_safe_path(extract_to, normalized_path):
-            extract_zip_member(zip_ref, member, normalized_path)
+            pairs.append((member, normalized_path))
         else:
             print(f"Skipping unsafe file {member.filename}")
+    _extract_members_staged(zip_ref, pairs)
 
 
 def extract_with_structure(zip_ref, extract_to, protected_files):
     print(f"Extracting with structure to {extract_to}")
     protected_files_lower = {f.lower() for f in protected_files}
+    pairs = []
     for member in zip_ref.infolist():
         target_path = os.path.join(extract_to, member.filename)
         normalized_path = os.path.normpath(target_path)
@@ -248,45 +256,200 @@ def extract_with_structure(zip_ref, extract_to, protected_files):
         if member.is_dir():
             os.makedirs(normalized_path, exist_ok=True)
             continue
-        extract_zip_member(zip_ref, member, normalized_path)
+        pairs.append((member, normalized_path))
+    _extract_members_staged(zip_ref, pairs)
 
 
-def extract_zip_member(zip_ref, member, target_path):
-    os.makedirs(os.path.dirname(target_path), exist_ok=True)
-    max_retries = 3
-    retry_delay = 10  # seconds
+def _extract_members_staged(zip_ref, pairs) -> None:
+    """Stage every member to a sibling ``.rfnew``, then swap all in."""
+    # duplicate targets, last member wins
+    fold = str.lower if sys.platform == "darwin" else os.path.normcase
+    deduped = {fold(target): (member, target) for member, target in pairs}
+    staged = []  # (tmp_path, target_path)
+    try:
+        for member, target_path in deduped.values():
+            os.makedirs(os.path.dirname(target_path), exist_ok=True)
+            if _member_matches_disk(zip_ref, member, target_path):
+                continue
+            # sibling temp: same volume, so the swap is an atomic rename
+            tmp_path = target_path + ".rfnew"
+            try:
+                with zip_ref.open(member) as source, open(tmp_path, 'wb') as target:
+                    shutil.copyfileobj(source, target)
+            except BaseException:
+                _remove_if_exists(tmp_path)
+                raise
+            staged.append((tmp_path, target_path))
+    except BaseException as e:  # cancellation must clean up too
+        for tmp_path, _ in staged:
+            _remove_if_exists(tmp_path)
+        if isinstance(e, Exception):
+            print(f"Extraction failed while staging; no files were changed: {e}")
+        raise
 
-    for attempt in range(max_retries):
+    failed = []
+    for tmp_path, target_path in staged:
         try:
-            with zip_ref.open(member) as source, open(target_path, 'wb') as target:
-                shutil.copyfileobj(source, target)
-            return  # Successful extraction, exit the function
+            displaced = _swap_into_place(tmp_path, target_path)
         except PermissionError:
-            file_name = os.path.basename(target_path)
-            folder_path = os.path.dirname(target_path)
-            
-            error_msg = [
-                f"\nPermission Error: Unable to extract {file_name}",
-                "\nThis could be because:",
-                "1. The file is currently in use by another program (e.g., MacroQuest, EQBCS)",
-                "2. You don't have write permissions for this location",
-                "\nPossible solutions:",
-                "1. Close all EverQuest-related programs (MacroQuest, EQBCS, etc.)",
-                f"2. Change the installation directory in settings to a location you own",
-                f"3. Manually set write permissions on: {folder_path}",
-            ]
-            
-            if attempt < max_retries - 1:
-                error_msg.append(f"\nRetrying in {retry_delay} seconds... (Attempt {attempt + 1} of {max_retries})")
-                print("\n".join(error_msg))
-                time.sleep(retry_delay)
-            else:
-                error_msg.append("\nMaximum retry attempts reached. Please resolve the permission issue and try again.")
-                print("\n".join(error_msg))
-                raise PermissionError(f"Failed to extract {file_name} after {max_retries} attempts.")
-        except Exception as e:
-            print(f"Unexpected error while extracting {os.path.basename(target_path)}: {str(e)}")
-            raise
+            _report_locked_target(target_path)
+            failed.append(target_path)
+            continue
+        except OSError as e:  # e.g. AV quarantined the .rfnew; don't abort the rest
+            print(f"Could not update {os.path.basename(target_path)}: {e}")
+            _remove_if_exists(tmp_path)
+            failed.append(target_path)
+            continue
+        if displaced:
+            print(f"Staged update for {os.path.basename(target_path)}; applies on next launch.")
+    if failed:
+        names = ", ".join(os.path.basename(path) for path in failed)
+        raise OSError(f"Could not update file(s): {names}")
+
+
+def _swap_into_place(tmp_path: str, target_path: str) -> bool:
+    """On Windows, files that are in use (like loaded executables) can't be overwritten directly. 
+    Instead, we rename the locked file out of the way and put the new one in its place."""
+    try:
+        os.replace(tmp_path, target_path)
+        return False
+    except PermissionError:
+        pass
+
+    if os.path.isdir(target_path):
+        # a directory shadowing a file member
+        _remove_if_exists(tmp_path)
+        raise IsADirectoryError(f"Target is a directory: {target_path}")
+
+    # a read-only target fails the same way a locked one does
+    cleared_readonly = _clear_readonly(target_path)
+    if cleared_readonly:
+        try:
+            os.replace(tmp_path, target_path)
+            return False
+        except PermissionError:
+            pass
+
+    old_path = _displacement_path(target_path)
+    try:
+        os.replace(target_path, old_path)
+    except PermissionError:
+        if cleared_readonly:
+            _restore_readonly(target_path)
+        _remove_if_exists(tmp_path)
+        raise
+    _mark_fresh(old_path)  # rename kept the old mtime; a concurrent sweep would eat it
+    try:
+        os.replace(tmp_path, target_path)
+    except BaseException:
+        # roll back so the target never goes missing
+        try:
+            os.replace(old_path, target_path)
+        except OSError:
+            pass
+        else:
+            if cleared_readonly:
+                _restore_readonly(target_path)
+        _remove_if_exists(tmp_path)
+        raise
+    return True
+
+
+def _displacement_path(target_path: str) -> str:
+    """Pick a free name for the displaced old file."""
+    for n in range(1000):
+        candidate = f"{target_path}.rfold{n or ''}"
+        _remove_if_exists(candidate)
+        if not os.path.exists(candidate):
+            return candidate
+    return candidate  # 1000 stuck generations: let the swap fail loudly
+
+
+def _member_matches_disk(zip_ref, member, target_path: str) -> bool:
+    """checks if the file on disk matches the zip member, to skip the swap"""
+    try:
+        if not os.path.isfile(target_path) or os.path.getsize(target_path) != member.file_size:
+            return False
+        crc = 0
+        with open(target_path, 'rb') as existing:
+            for chunk in iter(lambda: existing.read(262_144), b""):
+                crc = zlib.crc32(chunk, crc)
+        return crc == member.CRC
+    except OSError:
+        return False
+
+
+def _clear_readonly(path: str) -> bool:
+    """True if *path* was read-only and the attribute was cleared."""
+    try:
+        if os.access(path, os.W_OK):
+            return False
+        os.chmod(path, stat.S_IWRITE)
+        return True
+    except OSError:
+        return False
+
+
+def _restore_readonly(path: str) -> None:
+    """Put back a read-only bit we cleared for a swap that then failed."""
+    try:
+        os.chmod(path, stat.S_IREAD)
+    except OSError:
+        pass
+
+
+def _mark_fresh(path: str) -> None:
+    """Stamp now so the mtime sweep guard sees an in-flight file, not stale debris."""
+    try:
+        os.utime(path)
+    except OSError:
+        pass
+
+
+def _remove_if_exists(path: str) -> None:
+    try:
+        os.remove(path)
+    except PermissionError:
+        # the read-only bit survives displacement renames and blocks deletion
+        try:
+            os.chmod(path, stat.S_IWRITE)
+            os.remove(path)
+        except OSError:
+            pass
+    except OSError:
+        pass
+
+
+_STALE_DEBRIS_AGE = 3600  # seconds
+
+
+def sweep_stale_swap_files(directory: str) -> None:
+    """Recursively remove .rfnew/.rfold debris left by prior runs."""
+    cutoff = time.time() - _STALE_DEBRIS_AGE
+    root = Path(directory)
+    # rglob does not interpret glob wildcards within directory names, e.g. "Very[Vanilla]MQ" is matched literally
+    for pattern in ("*.rfnew", "*.rfold*"):
+        for path in root.rglob(pattern):
+            try:
+                if os.path.getmtime(path) >= cutoff:
+                    continue
+            except OSError:
+                continue
+            _remove_if_exists(str(path))
+
+
+def _report_locked_target(target_path: str) -> None:
+    """Friendly report for swap failure."""
+    file_name = os.path.basename(target_path)
+    folder_path = os.path.dirname(target_path)
+    print("\n".join([
+        f"\nPermission Error: Unable to update {file_name}",
+        "\nThe file is held open by another program and can't be replaced.",
+        "\nPossible solutions:",
+        "1. Close the program holding it open (e.g. one keeping a database file open)",
+        "2. Change the installation directory in settings to a location you own",
+        f"3. Manually set write permissions on: {folder_path}",
+    ]))
 
 
 def delete_zip_file(zip_path):

{redfetch-1.3.0 → redfetch-1.4.0}/src/redfetch/listener.py

@@ -89,9 +89,14 @@ async def handle_download(request: web.Request) -> web.Response:
     try:
         resource_id_str = str(resource_id)
         db_path = store.get_db_path(db_name)
-        success = await sync.run_sync(db_path, headers, resource_ids=[resource_id_str])
-        if success:
+        outcome = await sync.run_sync(db_path, headers, resource_ids=[resource_id_str])
+        if outcome.success:
             return web.json_response({"success": True, "message": "Download completed successfully."})
+        if outcome.status == "busy":
+            return web.json_response(
+                {"success": False, "message": "Another update is already in progress; please retry shortly."},
+                status=503, headers={"Retry-After": "5"},
+            )
         return web.json_response({"success": False, "message": "Download failed due to internal error."}, status=500)
     except Exception as e:
         log.error("Error during download: %s", e, exc_info=True)
@@ -105,11 +110,16 @@ async def handle_download_watched(request: web.Request) -> web.Response:
 
     try:
         db_path = store.get_db_path(db_name)
-        success = await sync.run_sync(db_path, headers)
-        if success:
+        outcome = await sync.run_sync(db_path, headers)
+        if outcome.success:
             return web.json_response(
                 {"success": True, "message": "All watched resources downloaded successfully."}
             )
+        if outcome.status == "busy":
+            return web.json_response(
+                {"success": False, "message": "Another update is already in progress; please retry shortly."},
+                status=503, headers={"Retry-After": "5"},
+            )
         return web.json_response(
             {"success": False, "message": "Download of one or more resources failed."}, status=500
         )