arelle-release 2.17.1__py3-none-any.whl → 2.37.71__py3-none-any.whl

arelle/WebCache.py

@@ -6,20 +6,42 @@ e.g., User-Agent: Sample Company Name AdminContact@<sample company domain>.com
 
 '''
 from __future__ import annotations
-from typing import TYPE_CHECKING, Any, Optional
-import os, posixpath, sys, time, calendar, io, json, logging, shutil, zlib
-import regex as re
-from urllib.parse import quote, unquote
-from urllib.error import URLError, HTTPError, ContentTooShortError
+
+import calendar
+import contextlib
+import io
+import json
+import logging
+import os
+import posixpath
+import shutil
+import sys
+import time
+import zlib
 from http.client import IncompleteRead
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
 from urllib import request as proxyhandlers
+from urllib.error import ContentTooShortError, HTTPError, URLError
+from urllib.parse import quote, unquote, urlsplit, urlunsplit
 
 import certifi
+import regex as re
+from filelock import FileLock, Timeout
+
+from arelle.PythonUtil import isLegacyAbs
 
 try:
     import ssl
 except ImportError:
     ssl = None
+
+try:
+    import truststore
+except ImportError:
+    # truststore requires Python > 3.9
+    truststore = None
+
 from arelle.FileSource import SERVER_WEB_CACHE, archiveFilenameParts
 from arelle.PluginManager import pluginClassMethods
 from arelle.UrlUtil import isHttpUrl
@@ -31,9 +53,23 @@ if TYPE_CHECKING:
 addServerWebCache = None
 
 DIRECTORY_INDEX_FILE = "!~DirectoryIndex~!"
+FILE_LOCK_TIMEOUT = 30
 INF = float("inf")
 RETRIEVAL_RETRY_COUNT = 5
-HTTP_USER_AGENT = 'Mozilla/5.0 (Arelle/{})'.format(__version__)
+HTTP_USER_AGENT = 'Mozilla/5.0 (Arelle/{}) Email/NotRegistered@arelle.org'.format(__version__)
+
+# The xbrl.org server accepts requests for both http and https as well as with or without the WWW subdomain.
+# Don't require duplicating these files in the cache.
+_XBRL_ORG_URL_PREFIXES = frozenset([
+    "http://www.xbrl.org/",
+    "http://xbrl.org/",
+    "https://www.xbrl.org/",
+    "https://xbrl.org/",
+])
+XBRL_ORG_CACHE_REDIRECTS = {
+    prefix: _XBRL_ORG_URL_PREFIXES.difference({prefix})
+    for prefix in _XBRL_ORG_URL_PREFIXES
+}
 
 def proxyDirFmt(httpProxyTuple):
     if isinstance(httpProxyTuple,(tuple,list)) and len(httpProxyTuple) == 5:
@@ -94,6 +130,7 @@ class WebCache:
         self._noCertificateCheck = False
         self._httpUserAgent = HTTP_USER_AGENT # default user agent for product
         self._httpsRedirect = False
+        self._redirectFallbackMap = {}
         self.resetProxies(httpProxyTuple)
 
         self.opener.addheaders = [('User-agent', self.httpUserAgent)]
@@ -113,6 +150,7 @@ class WebCache:
                 self.encodeFileChars = re.compile(r'[<>:"\\|?*^]')
             else:
                 self.encodeFileChars = re.compile(r'[:^]')
+        self.builtInCacheDir = os.path.abspath(os.path.join(__file__, '..', 'resources', 'cache'))
         self.decodeFileChars = re.compile(r'\^[0-9]{3}')
         self.workOffline: bool = False
         self._logDownloads = False
@@ -206,6 +244,9 @@ class WebCache:
     def httpsRedirect(self, value):
         self._httpsRedirect = value
 
+    def redirectFallback(self, matchPattern: re.Pattern, replaceFormat: str):
+        self._redirectFallbackMap[matchPattern] = replaceFormat
+
     def resetProxies(self, httpProxyTuple):
         # for ntlm user and password are required
         self.hasNTLM = False
@@ -241,12 +282,10 @@ class WebCache:
             self.http_auth_handler = proxyhandlers.HTTPBasicAuthHandler()
             proxyHandlers = [self.proxy_handler, self.proxy_auth_handler, self.http_auth_handler]
         if ssl:
-            # Attempts to load the default CA certificates from the OS.
-            context = ssl.create_default_context()
+            # Attempt to load the default CA certificates from the OS using truststore if available, else fallback to OpenSSL's default context.
+            context = truststore.SSLContext(ssl.PROTOCOL_TLS_CLIENT) if truststore else ssl.create_default_context()
             # Include certifi certificates (Mozilla’s carefully curated
-            # collection) for systems with outdated certs and for platforms
-            # that we're unable to load certs from (macOS and some Linux
-            # distros.)
+            # collection) for systems with outdated certs.
             context.load_verify_locations(cafile=certifi.where())
             if self.noCertificateCheck:  # this is required in some Akamai environments, such as sec.gov
                 context.check_hostname = False
@@ -261,11 +300,37 @@ class WebCache:
         #self.opener.close()
         #self.opener = WebCacheUrlOpener(self.cntlr, proxyDirFmt(httpProxyTuple))
 
-    def normalizeUrl(self, url: Optional[str], base: Optional[str] = None) -> Any:
+    def normalizeFilepath(self, filepath: str, url: str, cacheDir: str = None) -> str:
+        """
+        Perform any necessary transformations to filepath.
+        :param filepath: Filepath to normalize.
+        :param url: Original URL (for http/https redirect).
+        :param cacheDir: Cache root directory.
+        :return: Normalized filepath.
+        """
+        if cacheDir is None:
+            cacheDir = self.cacheDir
+        if self.httpsRedirect:
+            if not os.path.exists(filepath):
+                # if enabled, check for missing files in their inverse http/https cache directory
+                redirect = None
+                if url.startswith('http://'):
+                    redirect = self.urlToCacheFilepath('https' + url[4:], cacheDir=cacheDir)
+                elif url.startswith('https://'):
+                    redirect = self.urlToCacheFilepath('http' + url[5:], cacheDir=cacheDir)
+                if redirect and os.path.exists(redirect):
+                    filepath = redirect
+        if filepath.endswith("/"):
+            filepath += DIRECTORY_INDEX_FILE
+        if os.sep == '\\':
+            filepath = filepath.replace('/', '\\')
+        return filepath
+
+    def normalizeUrl(self, url: str | None, base: str | None = None) -> Any:
         if url:
             if url.startswith("file://"): url = url[7:]
             elif url.startswith("file:\\"): url = url[6:]
-        if url and not (isHttpUrl(url) or os.path.isabs(url)):
+        if url and not (isHttpUrl(url) or isLegacyAbs(url)):
             if base is not None and not isHttpUrl(base) and '%' in url:
                 url = unquote(url)
             if base:
@@ -282,7 +347,7 @@ class WebCache:
             elif normedPath.startswith("file:\\"): normedPath = normedPath[6:]
 
             # no base, not normalized, must be relative to current working directory
-            if base is None and not os.path.isabs(url):
+            if base is None and not isLegacyAbs(url):
                 normedPath = os.path.abspath(normedPath)
         else:
             normedPath = url
@@ -301,9 +366,70 @@ class WebCache:
     def encodeForFilename(self, pathpart):
         return self.encodeFileChars.sub(lambda m: '^{0:03}'.format(ord(m.group(0))), pathpart)
 
-    def urlToCacheFilepath(self, url: str) -> str:
+    def _fallbackRedirect(self, url: str, originalFilepath: str, cacheDir: str) -> str:
+        """
+        If the original URL does not map to an existing cache file,
+        we'll check each fallback redirect pattern to see if modifying
+        the URL yields a path to a file that does exist in the cache.
+        If none is found, the original filepath is returned.
+        :param url: The requested URL.
+        :param originalFilepath: The original mapped filepath.
+        :return: An existing redirected path or the original filepath.
+        """
+        if os.path.exists(originalFilepath):
+            return originalFilepath
+        for fromPattern, toPattern in self._redirectFallbackMap.items():
+            match = fromPattern.match(url)
+            if not match:
+                continue
+            redirectUrl = toPattern.format(*match[1:])
+            redirectFilepath = self.urlToCacheFilepath(
+                redirectUrl,
+                cacheDir,
+                useRedirectFallback=False  # prevent infinite recursion
+            )
+            if os.path.exists(redirectFilepath):
+                if self.cntlr.modelManager.modelXbrl:
+                    self.cntlr.modelManager.modelXbrl.warning(
+                        codes='arelle:redirectedUrl',
+                        msg='Redirected URL from %(url)s to %(redirectUrl)s',
+                        args={'url': url, 'redirectUrl': redirectUrl},
+                        file=redirectUrl
+                    )
+                else:
+                    self.cntlr.addToLog(
+                        messageCode='arelle:redirectedUrl',
+                        message='Redirected URL from %(url)s to %(redirectUrl)s',
+                        messageArgs={'url': url, 'redirectUrl': redirectUrl},
+                        file=redirectUrl,
+                        level=logging.WARNING,
+                    )
+                return redirectFilepath
+        for fromUrlPrefix, toUrlPrefixes in XBRL_ORG_CACHE_REDIRECTS.items():
+            if url.startswith(fromUrlPrefix):
+                for toUrlPrefix in toUrlPrefixes:
+                    redirectFilepath = self.urlToCacheFilepath(
+                        toUrlPrefix + url[len(fromUrlPrefix):],
+                        cacheDir,
+                        useRedirectFallback=False
+                    )
+                    if os.path.exists(redirectFilepath):
+                        return redirectFilepath
+                break
+        return originalFilepath
+
+    def urlToCacheFilepath(self, url: str, cacheDir: str | None = None, useRedirectFallback: bool = True) -> str:
+        """
+        Converts `url` into the corresponding cache filepath in `cacheDir.
+        :param url: URL to convert.
+        :param cacheDir: Cache root directory.
+        :param useRedirectFallback: Whether to use fallback redirects.
+        :return: Cache filepath.
+        """
+        if cacheDir is None:
+            cacheDir = self.cacheDir
         scheme, sep, path = url.partition("://")
-        filepath = [self.cacheDir, scheme]
+        filepath = [cacheDir, scheme]
         pathparts = path.split('/')
         user, sep, server = pathparts[0].partition("@")
         if not sep:
@@ -318,10 +444,15 @@ class WebCache:
         filepath.extend(self.encodeForFilename(pathpart) for pathpart in pathparts[1:])
         if url.endswith("/"):  # default index file
             filepath.append(DIRECTORY_INDEX_FILE)
-        return os.sep.join(filepath)
-
-    def cacheFilepathToUrl(self, cacheFilepath):
-        urlparts = cacheFilepath[len(self.cacheDir)+1:].split(os.sep)
+        joined_filepath = os.sep.join(filepath)
+        if useRedirectFallback:
+            return self._fallbackRedirect(url, joined_filepath, cacheDir)
+        return joined_filepath
+
+    def cacheFilepathToUrl(self, cacheFilepath: str, cacheDir: str | None = None) -> str:
+        if cacheDir is None:
+            cacheDir = self.cacheDir
+        urlparts = cacheFilepath[len(cacheDir)+1:].split(os.sep)
         urlparts[0] += ':/'  # add separator between http and file parts, less one '/'
         if len(urlparts) > 2:
             if urlparts[2].startswith("^port"):
@@ -339,271 +470,377 @@ class WebCache:
     def getfilename(
             self, url: str | None, base: str | None = None,
             reload: bool = False, checkModifiedTime: bool = False,
-            normalize: bool = False, filenameOnly: bool = False) -> str | None:
+            normalize: bool = False, filenameOnly: bool = False,
+            allowTransformation: bool = True) -> str | None:
+        if allowTransformation:
+            for pluginXbrlMethod in pluginClassMethods("WebCache.TransformURL"):
+                url, final = pluginXbrlMethod(self.cntlr, url, base)
+                if final:
+                    return url
         if url is None:
             return url
         if base is not None or normalize:
             url = self.normalizeUrl(url, base)
+
+        # Handle non-http, non-https URLS
         urlScheme, schemeSep, urlSchemeSpecificPart = url.partition("://")
-        if schemeSep and urlScheme in ("http", "https"):
-            # is this a mapped archive file contents?
-            _archiveFileNameParts = archiveFilenameParts(url)
-            if _archiveFileNameParts:
-                _archiveFilename = self.getfilename(_archiveFileNameParts[0], reload=reload, checkModifiedTime=checkModifiedTime)
-                if _archiveFilename:
-                    return os.path.join(_archiveFilename, _archiveFileNameParts[1])
-                return None
-            # form cache file name (substituting _ for any illegal file characters)
-            filepath = self.urlToCacheFilepath(url)
-            if self.httpsRedirect:
-                if not os.path.exists(filepath):
-                    # if enabled, check for missing files in their inverse http/https cache directory
-                    redirect = None
-                    if url.startswith('http://'):
-                        redirect = self.urlToCacheFilepath('https' + url[4:])
-                    elif url.startswith('https://'):
-                        redirect = self.urlToCacheFilepath('http' + url[5:])
-                    if redirect and os.path.exists(redirect):
-                        filepath = redirect
-            if self.cacheDir == SERVER_WEB_CACHE:
-                # server web-cached files are downloaded when opening to prevent excessive memcache api calls
-                return filepath
-            # quotedUrl has scheme-specific-part quoted except for parameter separators
-            quotedUrl = urlScheme + schemeSep + quote(urlSchemeSpecificPart, '/?=&')
-            # handle default directory requests
-            if filepath.endswith("/"):
-                filepath += DIRECTORY_INDEX_FILE
+        if not schemeSep or urlScheme not in ("http", "https"):
+            if url.startswith("file://"): url = url[7:]
+            elif url.startswith("file:\\"): url = url[6:]
             if os.sep == '\\':
-                filepath = filepath.replace('/', '\\')
-            if self.workOffline or filenameOnly:
-                return filepath
-            filepathtmp = filepath + ".tmp"
-            fileExt = os.path.splitext(filepath)[1]
-            timeNow = time.time()
-            timeNowStr = time.strftime('%Y-%m-%dT%H:%M:%S UTC', time.gmtime(timeNow))
-            retrievingDueToRecheckInterval = False
-            if not reload and os.path.exists(filepath):
-                if url in self.cachedUrlCheckTimes and not checkModifiedTime:
-                    cachedTime = calendar.timegm(time.strptime(self.cachedUrlCheckTimes[url], '%Y-%m-%dT%H:%M:%S UTC'))
-                else:
-                    cachedTime = 0
-                if timeNow - cachedTime > self.maxAgeSeconds:
-                    # weekly check if newer file exists
-                    newerOnWeb = False
-                    try: # no provision here for proxy authentication!!!
-                        remoteFileTime = lastModifiedTime( self.getheaders(quotedUrl) )
-                        if remoteFileTime and remoteFileTime > os.path.getmtime(filepath):
-                            newerOnWeb = True
-                    except:
-                        pass # for now, forget about authentication here
-                    if not newerOnWeb:
-                        # update ctime by copying file and return old file
-                        self.cachedUrlCheckTimes[url] = timeNowStr
-                        self.cachedUrlCheckTimesModified = True
-                        return filepath
-                    retrievingDueToRecheckInterval = True
+                url = url.replace('/', '\\')
+            return url
+
+        # Handle archive URLs with recursive call
+        _archiveFileNameParts = archiveFilenameParts(url)
+        if _archiveFileNameParts:
+            _archiveFilename = self.getfilename(_archiveFileNameParts[0], reload=reload, checkModifiedTime=checkModifiedTime)
+            if _archiveFilename:
+                return os.path.join(_archiveFilename, _archiveFileNameParts[1])
+            return None
+
+        # Generate cache filepath from url
+        filepath = self.urlToCacheFilepath(url)
+
+        # Handle server web cache
+        if self.cacheDir == SERVER_WEB_CACHE:
+            # server web-cached files are downloaded when opening to prevent excessive memcache api calls
+            return filepath
+
+        # Make any necessary transformations to filepath
+        filepath = self.normalizeFilepath(filepath, url)
+        filepathExists = os.path.exists(filepath)
+
+        # If cache filepath does not exist, try built-in cache
+        if self.workOffline and not filepathExists:
+            builtInFilepath = self.normalizeFilepath(self.urlToCacheFilepath(url, cacheDir=self.builtInCacheDir), url)
+            if os.path.exists(builtInFilepath):
+                return builtInFilepath
+
+        # Return filepath now if configured not to download
+        if self.workOffline or filenameOnly:
+            return filepath
+
+        # Download without checking age if configured to do so or file does not exist
+        if reload or not filepathExists:
+            return filepath if self._downloadFileWithLock(url, filepath) else None
+
+        # Determine if file has aged out of cache, return filepath if not
+        if url in self.cachedUrlCheckTimes and not checkModifiedTime:
+            cachedTime = calendar.timegm(time.strptime(self.cachedUrlCheckTimes[url], '%Y-%m-%dT%H:%M:%S UTC'))
+        else:
+            cachedTime = 0
+        timeNow = time.time()
+        if timeNow - cachedTime <= self.maxAgeSeconds:
+            return filepath
+
+        # If we determine that the web version is newer, download it
+        if self._checkIfNewerOnWeb(url, filepath):
+            self._downloadFileWithLock(url, filepath, retrievingDueToRecheckInterval=True)
+            # Whether the download is successful or not, we know `filepath` exists, so return it.
+            return filepath
+        # Otherwise, use existing file
+        self.cachedUrlCheckTimes[url] = WebCache._getTimeString(timeNow)
+        self.cachedUrlCheckTimesModified = True
+        return filepath
+
+    def _checkIfNewerOnWeb(self, url: str, filepath: str) -> bool:
+        """
+
+        :param url: URL to retrieve web timestamp from
+        :param filepath: Filepath to retrieve local timestamp from
+        :return:
+        """
+        # quotedUrl has scheme-specific-part quoted except for parameter separators
+        quotedUrl = WebCache._quotedUrl(url)
+        try:  # no provision here for proxy authentication!!!
+            remoteFileTime = lastModifiedTime(self.getheaders(quotedUrl))
+            if remoteFileTime and remoteFileTime > os.path.getmtime(filepath):
+                return True
+        except:
+            pass  # for now, forget about authentication here
+        return False
+
+    @staticmethod
+    def _getTimeString(timeValue: time.time) -> str:
+        """
+        :param timeValue:
+        :return: UTC-formatted string representation of `timeValue`
+        """
+        return time.strftime('%Y-%m-%dT%H:%M:%S UTC', time.gmtime(timeValue))
+
+    @staticmethod
+    def _quotedUrl(url: str) -> str:
+        """
+        :param url:
+        :return: `url` with scheme-specific-part quoted except for parameter separators
+        """
+        parts = urlsplit(url)
+
+        # RFC 3986 safe characters: https://www.ietf.org/rfc/rfc3986.txt
+        pathSafe = "/:@!$&'()*+,;="       # path allows sub-delims, ":" and "@"
+        querySafe = "&=:/?@!$'()*+,;[]"   # query allows pchar + "/" + "?"
+
+        quotedPath = quote(parts.path, safe=pathSafe)
+        quotedQuery = quote(parts.query, safe=querySafe)
+
+        return urlunsplit((parts.scheme, parts.netloc, quotedPath, quotedQuery, parts.fragment))
+
+    @staticmethod
+    def _getFileTimestamp(path: str) -> float:
+        try:
+            stats = Path(path).stat()
+            return max(stats.st_ctime, stats.st_mtime)
+        except FileNotFoundError:
+            return 0
+
+    def _downloadFileWithLock(
+            self,
+            url: str,
+            filepath: str,
+            retrievingDueToRecheckInterval: bool = False,
+            retryCount: int = 5) -> bool:
+        before_timestamp = WebCache._getFileTimestamp(filepath)
+        fileInCache = False
+        lock = FileLock(filepath + '.lock', timeout=FILE_LOCK_TIMEOUT)
+        try:
+            with lock.acquire():
+                after_timestamp = WebCache._getFileTimestamp(filepath)
+                if after_timestamp > before_timestamp:
+                    # Another process just downloaded the file, use it instead.
+                    fileInCache = True
                 else:
-                    return filepath
-            filedir = os.path.dirname(filepath)
-            if not os.path.exists(filedir):
-                os.makedirs(filedir)
-            # Retrieve over HTTP and cache, using rename to avoid collisions
-            # self.modelManager.addToLog('web caching: {0}'.format(url))
-
-            # download to a temporary name so it is not left readable corrupted if download fails
-            retryCount = 5
-            while retryCount > 0:
-                try:
-                    self.progressUrl = url
-                    savedfile, headers, initialBytes = self.retrieve(
+                    fileInCache = self._downloadFile(url, filepath, retrievingDueToRecheckInterval, retryCount)
+            # Clean up the lock file. The lock context manager releases the file lock but does not delete the lock file.
+            # There is a minor race condition when multiple Arelle processes run concurrently. One process may release
+            # its lock, and another may acquire it before the first process can delete the lock file. In this case, the
+            # first process will suppress the OSError and continue, while the second process will skip downloading the
+            # file because the file timestamp has updated, and then remove the lock file before returning.
+            with contextlib.suppress(OSError):
+                os.remove(lock.lock_file)
+        except Timeout:
+            self.cntlr.addToLog(_("Unable to obtain exclusive write access to download file. Delete file and retry: %(lockfile)s"),
+                                messageCode="webCache:cacheFileLocked",
+                                messageArgs={"lockfile": lock.lock_file},
+                                level=logging.ERROR)
+        return fileInCache
+
+    def _downloadFile(
+            self,
+            url: str,
+            filepath: str,
+            retrievingDueToRecheckInterval: bool = False,
+            retryCount: int = 5) -> bool:
+        """
+        Downloads the file at `url` to a temporary location before copying it to `filepath`.
+        :param url: Web resource to download.
+        :param filepath: End destination for downloaded file.
+        :param retrievingDueToRecheckInterval: Determines how errors are handled when download is part of a cache recheck.
+        :param retryCount: Number of times to retry download.
+        :return: Whether `filepath` should now be used.
+        """
+        temporaryFilename = Path(f'{filepath}.tmp')
+        fileExt = os.path.splitext(filepath)[1]
+        timeNowStr = WebCache._getTimeString(time.time())
+        quotedUrl = WebCache._quotedUrl(url)
+
+        filedir = os.path.dirname(filepath)
+        os.makedirs(filedir, exist_ok=True)
+        # Retrieve over HTTP and cache, using rename to avoid collisions
+        # self.modelManager.addToLog('web caching: {0}'.format(url))
+
+        # download to a temporary name so it is not left readable corrupted if download fails
+        while retryCount > 0:
+            try:
+                self.progressUrl = url
+                savedfile, headers, initialBytes = self.retrieve(
                     #savedfile, headers = self.opener.retrieve(
-                                      quotedUrl,
-                                      filename=filepathtmp,
-                                      reporthook=self.reportProgress)
-
-                    # check if this is a real file or a wifi or web logon screen
-                    if fileExt in {".xsd", ".xml", ".xbrl"}:
-                        if b"<html" in initialBytes:
-                            if retrievingDueToRecheckInterval:
-                                return self.internetRecheckFailedRecovery(filepath, url,
-                                                                          "file contents appear to be an html logon request",
-                                                                          timeNowStr)
-                            response = None  # found possible logon request
-                            if self.cntlr.hasGui:
-                                response = self.cntlr.internet_logon(url, quotedUrl,
-                                                                     _("Unexpected HTML in {0}").format(url),
-                                                                     _("Is this a logon page? If so, click 'yes', else click 'no' if it is the expected XBRL content, or 'cancel' to abort retrieval: \n\n{0}")
-                                                                     .format(initialBytes[:1500]))
-                            if response == "retry":
-                                retryCount -= 1
-                                continue
-                            elif response != "no":
-                                self.cntlr.addToLog(_("Web file appears to be an html logon request, not retrieved: %(URL)s \nContents: \n%(contents)s"),
-                                                    messageCode="webCache:invalidRetrieval",
-                                                    messageArgs={"URL": url, "contents": initialBytes},
-                                                    level=logging.ERROR)
-                                return None
+                    quotedUrl,
+                    filename=str(temporaryFilename),
+                    reporthook=self.reportProgress)
 
-                    retryCount = 0
-                except (ContentTooShortError, IncompleteRead) as err:
-                    if retrievingDueToRecheckInterval:
-                        return self.internetRecheckFailedRecovery(filepath, url, err, timeNowStr)
-                    if retryCount > 1:
-                        self.cntlr.addToLog(_("%(error)s \nunsuccessful retrieval of %(URL)s \n%(retryCount)s retries remaining"),
-                                            messageCode="webCache:retryingOperation",
-                                            messageArgs={"error": err, "URL": url, "retryCount": retryCount},
-                                            level=logging.ERROR)
-                        retryCount -= 1
-                        continue
-                    self.cntlr.addToLog(_("%(error)s \nretrieving %(URL)s"),
-                                        messageCode="webCache:contentTooShortError",
-                                        messageArgs={"URL": url, "error": err},
-                                        level=logging.ERROR)
-                    if os.path.exists(filepathtmp):
-                        os.remove(filepathtmp)
-                    return None
-                    # handle file is bad
-                except (HTTPError, URLError) as err:
-                    try:
-                        tryWebAuthentication = False
-                        if isinstance(err, HTTPError) and err.code == 401:
-                            tryWebAuthentication = True
-                            if 'www-authenticate' in err.hdrs:
-                                match = re.match('[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"', err.hdrs['www-authenticate'])
-                                if match:
-                                    scheme, realm = match.groups()
-                                    if scheme.lower() == 'basic':
-                                        host = os.path.dirname(quotedUrl)
-                                        userPwd = self.cntlr.internet_user_password(host, realm)
-                                        if isinstance(userPwd,(tuple,list)):
-                                            self.http_auth_handler.add_password(realm=realm,uri=host,user=userPwd[0],passwd=userPwd[1])
-                                            retryCount -= 1
-                                            continue
-                                    self.cntlr.addToLog(_("'%(scheme)s' www-authentication for realm '%(realm)s' is required to access %(URL)s\n%(error)s"),
-                                                        messageCode="webCache:unsupportedWWWAuthentication",
-                                                        messageArgs={"scheme": scheme, "realm": realm, "URL": url, "error": err},
-                                                        level=logging.ERROR)
-                        elif isinstance(err, HTTPError) and err.code == 407:
-                            tryWebAuthentication = True
-                            if 'proxy-authenticate' in err.hdrs:
-                                match = re.match('[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"', err.hdrs['proxy-authenticate'])
-                                if match:
-                                    scheme, realm = match.groups()
-                                    host = self.proxy_handler.proxies.get('http')
-                                    if scheme.lower() == 'basic':
-                                        userPwd = self.cntlr.internet_user_password(host, realm)
-                                        if isinstance(userPwd,(tuple,list)):
-                                            self.proxy_auth_handler.add_password(realm=realm,uri=host,user=userPwd[0],passwd=userPwd[1])
-                                            retryCount -= 1
-                                            continue
-                                    self.cntlr.addToLog(_("'%(scheme)s' proxy-authentication for realm '%(realm)s' is required to access %(URL)s\n%(error)s"),
-                                                        messageCode="webCache:unsupportedProxyAuthentication",
-                                                        messageArgs={"scheme": scheme, "realm": realm, "URL": url, "error": err},
-                                                        level=logging.ERROR)
+                # check if this is a real file or a wifi or web logon screen
+                if fileExt in {".xsd", ".xml", ".xbrl"}:
+                    if b"<html" in initialBytes:
                         if retrievingDueToRecheckInterval:
-                            return self.internetRecheckFailedRecovery(filepath, url, err, timeNowStr)
-                        if tryWebAuthentication:
-                            # check if single signon is requested (on first retry)
-                            if retryCount == RETRIEVAL_RETRY_COUNT:
-                                for pluginXbrlMethod in pluginClassMethods("Proxy.HTTPAuthenticate"):
-                                    if pluginXbrlMethod(self.cntlr): # true if succeessful single sign on
+                            err = "file contents appear to be an html logon request"
+                            self.internetRecheckFailedRecovery(url, err, timeNowStr)
+                            return True
+                        response = None  # found possible logon request
+                        if self.cntlr.hasGui:
+                            response = self.cntlr.internet_logon(url, quotedUrl,
+                                                                 _("Unexpected HTML in {0}").format(url),
+                                                                 _("Is this a logon page? If so, click 'yes', else click 'no' if it is the expected XBRL content, or 'cancel' to abort retrieval: \n\n{0}")
+                                                                 .format(initialBytes[:1500]))
+                        if response == "retry":
+                            retryCount -= 1
+                            continue
+                        elif response != "no":
+                            self.cntlr.addToLog(_("Web file appears to be an html logon request, not retrieved: %(URL)s \nContents: \n%(contents)s"),
+                                                messageCode="webCache:invalidRetrieval",
+                                                messageArgs={"URL": url, "contents": initialBytes},
+                                                level=logging.ERROR)
+                            return False
+
+                retryCount = 0
+            except (ContentTooShortError, IncompleteRead) as err:
+                if retrievingDueToRecheckInterval:
+                    self.internetRecheckFailedRecovery(url, err, timeNowStr)
+                    return True
+                if retryCount > 1:
+                    self.cntlr.addToLog(_("%(error)s \nunsuccessful retrieval of %(URL)s \n%(retryCount)s retries remaining"),
+                                        messageCode="webCache:retryingOperation",
+                                        messageArgs={"error": err, "URL": url, "retryCount": retryCount},
+                                        level=logging.ERROR)
+                    retryCount -= 1
+                    continue
+                self.cntlr.addToLog(_("%(error)s \nretrieving %(URL)s"),
+                                    messageCode="webCache:contentTooShortError",
+                                    messageArgs={"URL": url, "error": err},
+                                    level=logging.ERROR)
+                temporaryFilename.unlink(missing_ok=True)
+                return False
+                # handle file is bad
+            except (HTTPError, URLError) as err:
+                try:
+                    tryWebAuthentication = False
+                    if isinstance(err, HTTPError) and err.code == 401:
+                        tryWebAuthentication = True
+                        if 'www-authenticate' in err.hdrs:
+                            match = re.match('[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"', err.hdrs['www-authenticate'])
+                            if match:
+                                scheme, realm = match.groups()
+                                if scheme.lower() == 'basic':
+                                    host = os.path.dirname(quotedUrl)
+                                    userPwd = self.cntlr.internet_user_password(host, realm)
+                                    if isinstance(userPwd,(tuple,list)):
+                                        self.http_auth_handler.add_password(realm=realm,uri=host,user=userPwd[0],passwd=userPwd[1])
                                         retryCount -= 1
-                                        break
-                                if retryCount < RETRIEVAL_RETRY_COUNT:
-                                    continue # succeeded
-                            # may be a web login authentication request
-                            response = None  # found possible logon request
-                            if self.cntlr.hasGui:
-                                response = self.cntlr.internet_logon(url, quotedUrl,
-                                                                     _("HTTP {0} authentication request").format(err.code),
-                                                                     _("Is browser-based internet access authentication possible? If so, click 'yes', or 'cancel' to abort retrieval: \n\n{0}")
-                                                                     .format(url))
-                            if response == "retry":
-                                retryCount -= 1
-                                continue
-                            elif response != "no":
-                                self.cntlr.addToLog(_("Web file HTTP 401 (authentication required) response, not retrieved: %(URL)s"),
-                                                    messageCode="webCache:authenticationRequired",
-                                                    messageArgs={"URL": url},
+                                        continue
+                                self.cntlr.addToLog(_("'%(scheme)s' www-authentication for realm '%(realm)s' is required to access %(URL)s\n%(error)s"),
+                                                    messageCode="webCache:unsupportedWWWAuthentication",
+                                                    messageArgs={"scheme": scheme, "realm": realm, "URL": url, "error": err},
+                                                    level=logging.ERROR)
+                    elif isinstance(err, HTTPError) and err.code == 407:
+                        tryWebAuthentication = True
+                        if 'proxy-authenticate' in err.hdrs:
+                            match = re.match('[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"', err.hdrs['proxy-authenticate'])
+                            if match:
+                                scheme, realm = match.groups()
+                                host = self.proxy_handler.proxies.get('http')
+                                if scheme.lower() == 'basic':
+                                    userPwd = self.cntlr.internet_user_password(host, realm)
+                                    if isinstance(userPwd,(tuple,list)):
+                                        self.proxy_auth_handler.add_password(realm=realm,uri=host,user=userPwd[0],passwd=userPwd[1])
+                                        retryCount -= 1
+                                        continue
+                                self.cntlr.addToLog(_("'%(scheme)s' proxy-authentication for realm '%(realm)s' is required to access %(URL)s\n%(error)s"),
+                                                    messageCode="webCache:unsupportedProxyAuthentication",
+                                                    messageArgs={"scheme": scheme, "realm": realm, "URL": url, "error": err},
                                                     level=logging.ERROR)
-                                return None
-
-                    except AttributeError:
-                        pass
                     if retrievingDueToRecheckInterval:
-                        return self.internetRecheckFailedRecovery(filepath, url, err, timeNowStr)
-                    self.cntlr.addToLog(_("%(error)s \nretrieving %(URL)s"),
-                                        messageCode="webCache:retrievalError",
-                                        messageArgs={"error": err.reason if hasattr(err, "reason") else err,
-                                                     "URL": url},
+                        self.internetRecheckFailedRecovery(url, err, timeNowStr)
+                        return True
+                    if tryWebAuthentication:
+                        # check if single signon is requested (on first retry)
+                        if retryCount == RETRIEVAL_RETRY_COUNT:
+                            for pluginXbrlMethod in pluginClassMethods("Proxy.HTTPAuthenticate"):
+                                if pluginXbrlMethod(self.cntlr): # true if succeessful single sign on
+                                    retryCount -= 1
+                                    break
+                            if retryCount < RETRIEVAL_RETRY_COUNT:
+                                continue # succeeded
+                        # may be a web login authentication request
+                        response = None  # found possible logon request
+                        if self.cntlr.hasGui:
+                            response = self.cntlr.internet_logon(url, quotedUrl,
+                                                                 _("HTTP {0} authentication request").format(err.code),
+                                                                 _("Is browser-based internet access authentication possible? If so, click 'yes', or 'cancel' to abort retrieval: \n\n{0}")
+                                                                 .format(url))
+                        if response == "retry":
+                            retryCount -= 1
+                            continue
+                        elif response != "no":
+                            self.cntlr.addToLog(_("Web file HTTP 401 (authentication required) response, not retrieved: %(URL)s"),
+                                                messageCode="webCache:authenticationRequired",
+                                                messageArgs={"URL": url},
+                                                level=logging.ERROR)
+                            return False
+
+                except AttributeError:
+                    pass
+                if retrievingDueToRecheckInterval:
+                    self.internetRecheckFailedRecovery(url, err, timeNowStr)
+                    return True
+                self.cntlr.addToLog(_("%(error)s \nretrieving %(URL)s"),
+                                    messageCode="webCache:retrievalError",
+                                    messageArgs={"error": err.reason if hasattr(err, "reason") else err,
+                                                 "URL": url},
+                                    level=logging.ERROR)
+                return False
+
+            except Exception as err:
+                if retryCount > 1:
+                    self.cntlr.addToLog(_("%(error)s \nunsuccessful retrieval of %(URL)s \n%(retryCount)s retries remaining"),
+                                        messageCode="webCache:retryingOperation",
+                                        messageArgs={"error": err, "URL": url, "retryCount": retryCount},
                                         level=logging.ERROR)
-                    return None
-
-                except Exception as err:
-                    if retryCount > 1:
-                        self.cntlr.addToLog(_("%(error)s \nunsuccessful retrieval of %(URL)s \n%(retryCount)s retries remaining"),
-                                            messageCode="webCache:retryingOperation",
-                                            messageArgs={"error": err, "URL": url, "retryCount": retryCount},
-                                            level=logging.ERROR)
-                        retryCount -= 1
-                        continue
-                    if retrievingDueToRecheckInterval:
-                        return self.internetRecheckFailedRecovery(filepath, url, err, timeNowStr)
-                    if self.cntlr.hasGui:
-                        self.cntlr.addToLog(_("%(error)s \nunsuccessful retrieval of %(URL)s \nswitching to work offline"),
-                                            messageCode="webCache:attemptingOfflineOperation",
-                                            messageArgs={"error": err, "URL": url},
-                                            level=logging.ERROR)
-                        # try working offline
-                        self.workOffline = True
-                        return filepath
-                    else:  # don't switch offline unexpectedly in scripted (batch) operation
-                        self.cntlr.addToLog(_("%(error)s \nunsuccessful retrieval of %(URL)s"),
-                                            messageCode="webCache:unsuccessfulRetrieval",
-                                            messageArgs={"error": err, "URL": url},
-                                            level=logging.ERROR)
-                        if os.path.exists(filepathtmp):
-                            os.remove(filepathtmp)
-                        return None
-
-                # rename temporarily named downloaded file to desired name
-                if os.path.exists(filepath):
-                    try:
-                        if os.path.isfile(filepath) or os.path.islink(filepath):
-                            os.remove(filepath)
-                        elif os.path.isdir(filepath):
-                            shutil.rmtree(filepath)
-                    except Exception as err:
-                        self.cntlr.addToLog(_("%(error)s \nUnsuccessful removal of prior file %(filepath)s \nPlease remove with file manager."),
-                                            messageCode="webCache:cachedPriorFileLocked",
-                                            messageArgs={"error": err, "filepath": filepath},
-                                            level=logging.ERROR)
-                try:
-                    os.rename(filepathtmp, filepath)
-                    if self._logDownloads:
-                        self.cntlr.addToLog(_("Downloaded %(URL)s"),
-                                            messageCode="webCache:download",
-                                            messageArgs={"URL": url, "filepath": filepath},
-                                            level=logging.INFO)
-                except Exception as err:
-                    self.cntlr.addToLog(_("%(error)s \nUnsuccessful renaming of downloaded file to active file %(filepath)s \nPlease remove with file manager."),
-                                        messageCode="webCache:cacheDownloadRenamingError",
-                                        messageArgs={"error": err, "filepath": filepath},
+                    retryCount -= 1
+                    continue
+                if retrievingDueToRecheckInterval:
+                    self.internetRecheckFailedRecovery(url, err, timeNowStr)
+                    return True
+                if self.cntlr.hasGui:
+                    self.cntlr.addToLog(_("%(error)s \nunsuccessful retrieval of %(URL)s \nswitching to work offline"),
+                                        messageCode="webCache:attemptingOfflineOperation",
+                                        messageArgs={"error": err, "URL": url},
                                         level=logging.ERROR)
-                webFileTime = lastModifiedTime(headers)
-                if webFileTime: # set mtime to web mtime
-                    os.utime(filepath,(webFileTime,webFileTime))
-                self.cachedUrlCheckTimes[url] = timeNowStr
-                self.cachedUrlCheckTimesModified = True
-                return filepath
-
-        if url.startswith("file://"): url = url[7:]
-        elif url.startswith("file:\\"): url = url[6:]
-        if os.sep == '\\':
-            url = url.replace('/', '\\')
-        return url
+                    # try working offline
+                    self.workOffline = True
+                    return False
+                else:  # don't switch offline unexpectedly in scripted (batch) operation
+                    self.cntlr.addToLog(_("%(error)s \nunsuccessful retrieval of %(URL)s"),
+                                        messageCode="webCache:unsuccessfulRetrieval",
+                                        messageArgs={"error": err, "URL": url},
+                                        level=logging.ERROR)
+                    temporaryFilename.unlink(missing_ok=True)
+                    return False
 
-    def internetRecheckFailedRecovery(self, filepath, url, err, timeNowStr):
+            # rename temporarily named downloaded file to desired name
+            try:
+                if os.path.isdir(filepath):
+                    try:
+                        shutil.rmtree(filepath)
+                    except FileNotFoundError:
+                        pass
+                else:
+                    Path(filepath).unlink(missing_ok=True)
+            except Exception as err:
+                self.cntlr.addToLog(_("%(error)s \nUnsuccessful removal of prior file %(filepath)s \nPlease remove with file manager."),
+                                    messageCode="webCache:cachedPriorFileLocked",
+                                    messageArgs={"error": err, "filepath": filepath},
+                                    level=logging.ERROR)
+            try:
+                temporaryFilename.rename(filepath)
+                if self._logDownloads:
+                    self.cntlr.addToLog(_("Downloaded %(URL)s"),
+                                        messageCode="webCache:download",
+                                        messageArgs={"URL": url, "filepath": filepath},
+                                        level=logging.INFO)
+            except Exception as err:
+                self.cntlr.addToLog(_("%(error)s \nUnsuccessful renaming of downloaded file to active file %(filepath)s \nPlease remove with file manager."),
+                                    messageCode="webCache:cacheDownloadRenamingError",
+                                    messageArgs={"error": err, "filepath": filepath},
+                                    level=logging.ERROR)
+            webFileTime = lastModifiedTime(headers)
+            if webFileTime: # set mtime to web mtime
+                os.utime(filepath,(webFileTime,webFileTime))
+            self.cachedUrlCheckTimes[url] = timeNowStr
+            self.cachedUrlCheckTimesModified = True
+            return True
+        return False
+
+    def internetRecheckFailedRecovery(self, url: str, err: str | Exception, timeNowStr: str) -> None:
         self.cntlr.addToLog(_("During refresh of web file ignoring error: %(error)s for %(URL)s"),
                             messageCode="webCache:unableToRefreshFile",
                             messageArgs={"URL": url, "error": err},
@@ -611,7 +848,6 @@ class WebCache:
         # skip this checking cycle, act as if retrieval was ok
         self.cachedUrlCheckTimes[url] = timeNowStr
         self.cachedUrlCheckTimesModified = True
-        return filepath
 
     def reportProgress(self, blockCount, blockSize, totalSize):
         if totalSize > 0:
@@ -627,8 +863,10 @@ class WebCache:
     def clear(self):
         for cachedProtocol in ("http", "https"):
             cachedProtocolDir = os.path.join(self.cacheDir, cachedProtocol)
-            if os.path.exists(cachedProtocolDir):
-                shutil.rmtree(cachedProtocolDir, True)
+            try:
+                shutil.rmtree(cachedProtocolDir)
+            except FileNotFoundError:
+                pass
 
     def getheaders(self, url):
         if url and isHttpUrl(url):