arelle-release 2.17.1__py3-none-any.whl → 2.37.71__py3-none-any.whl

arelle/FileSource.py

@@ -2,30 +2,41 @@
 See COPYRIGHT.md for copyright information.
 '''
 from __future__ import annotations
-from typing import IO, TYPE_CHECKING, Any, Union, cast
-import zipfile, tarfile, os, io, errno, base64, gzip, zlib, struct, random
+
+import base64
+import errno
+import gzip
+import io
+import os
+import random
+import struct
+import tarfile
+import zipfile
+import zlib
+from typing import IO, TYPE_CHECKING, Any, BinaryIO, Optional, TextIO, cast
+
 import regex as re
 from lxml import etree
-from arelle import XmlUtil
-from arelle import PackageManager
-from arelle.UrlUtil import isHttpUrl
-from arelle.typing import TypeGetText
-import arelle.PluginManager
 
+import arelle.PluginManager
+from arelle import PackageManager, XmlUtil
+from arelle.PythonUtil import isLegacyAbs
+from arelle.packages.report.DetectReportPackage import isReportPackageExtension
+from arelle.packages.report.ReportPackage import ReportPackage
+from arelle.typing import TypeGetText
+from arelle.UrlUtil import isHttpUrl
 
 _: TypeGetText
 
 if TYPE_CHECKING:
     from arelle.Cntlr import Cntlr
-    from _typeshed import SupportsRead
 
 
-archivePathSeparators = (".zip" + os.sep, ".tar.gz" + os.sep, ".eis" + os.sep, ".xml" + os.sep, ".xfd" + os.sep, ".frm" + os.sep, '.taxonomyPackage.xml' + os.sep) + \
+archivePathSeparators = (".zip" + os.sep, ".xbr" + os.sep, ".xbri" + os.sep, ".tar.gz" + os.sep, ".eis" + os.sep, ".xml" + os.sep, ".xfd" + os.sep, ".frm" + os.sep, '.taxonomyPackage.xml' + os.sep) + \
                         ((".zip/", ".tar.gz/", ".eis/", ".xml/", ".xfd/", ".frm/", '.taxonomyPackage.xml/') if os.sep != "/" else ()) #acomodate windows and http styles
 
-archiveFilenameSuffixes = {".zip", ".tar.gz", ".eis", ".xml", ".xfd", ".frm"}
+archiveFilenameSuffixes = {".zip", ".xbr", ".xbri", ".tar.gz", ".eis", ".xml", ".xfd", ".frm"}
 
-POST_UPLOADED_ZIP = os.sep + "POSTupload.zip"
 SERVER_WEB_CACHE = os.sep + "_HTTP_CACHE"
 
 TAXONOMY_PACKAGE_FILE_NAMES = ('.taxonomyPackage.xml', 'catalog.xml') # pre-PWD packages
@@ -33,14 +44,21 @@ TAXONOMY_PACKAGE_FILE_NAMES = ('.taxonomyPackage.xml', 'catalog.xml') # pre-PWD
 def openFileSource(
     filename: str | None,
     cntlr: Cntlr | None = None,
-    sourceZipStream: str | None = None,
+    sourceZipStream: BinaryIO | FileNamedBytesIO | None = None,
     checkIfXmlIsEis: bool = False,
     reloadCache: bool = False,
     base: str | None = None,
     sourceFileSource: FileSource | None = None,
 ) -> FileSource:
     if sourceZipStream:
-        filesource = FileSource(POST_UPLOADED_ZIP, cntlr)
+        if name := getattr(sourceZipStream, "name", None):
+            # Python IO convention is to use the name attribute
+            sourceZipStreamFileName: str = os.sep + str(name)
+        elif isinstance(sourceZipStream, FileNamedBytesIO) and sourceZipStream.fileName:
+            sourceZipStreamFileName = os.sep + sourceZipStream.fileName
+        else:
+            sourceZipStreamFileName = os.sep + "POSTupload.zip"
+        filesource = FileSource(sourceZipStreamFileName, cntlr)
         filesource.openZipStream(sourceZipStream)
         if filename:
             filesource.select(filename)
@@ -56,13 +74,23 @@ def openFileSource(
             selection: str | None = archivepathSelection[1]
 
             assert selection is not None
+            selectionIsEmbeddedZip = False
             if (
                 sourceFileSource is not None
                 and sourceFileSource.dir is not None
                 and sourceFileSource.isArchive
                 and selection in sourceFileSource.dir
-                and selection.endswith(".zip")
             ):
+                if isReportPackageExtension(selection):
+                    selectionIsEmbeddedZip = True
+                else:
+                    try:
+                        assert isinstance(sourceFileSource.fs, zipfile.ZipFile)
+                        with sourceFileSource.fs.open(selection) as f:
+                            selectionIsEmbeddedZip = zipfile.is_zipfile(f)
+                    except Exception:
+                        pass
+            if selectionIsEmbeddedZip:
                 assert cntlr is not None
                 filesource = FileSource(filename, cntlr)
                 selection = None
@@ -85,24 +113,25 @@ def archiveFilenameParts(filename: str | None, checkIfXmlIsEis: bool = False) ->
             fileDir = filenameParts[0] + archiveSep[:-1]
             if (isHttpUrl(fileDir) or
                 os.path.isfile(fileDir)): # if local, be sure it is not a directory name
-                return (fileDir, filenameParts[2])
+                return (fileDir, filenameParts[2].replace('\\', '/'))
     return None
 
 class FileNamedStringIO(io.StringIO):  # provide string IO in memory but behave as a fileName string
     def __init__(self, fileName: str, *args: Any, **kwargs: Any) -> None:
-        super(FileNamedStringIO, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.fileName = fileName
 
     def close(self) -> None:
-        del self.fileName
-        super(FileNamedStringIO, self).close()
+        if hasattr(self, 'fileName'):
+            del self.fileName
+        super().close()
 
     def __str__(self) -> str:
         return self.fileName
 
 class FileNamedTextIOWrapper(io.TextIOWrapper):  # provide string IO in memory but behave as a fileName string
     def __init__(self, fileName: str, *args: Any, **kwargs: Any):
-        super(FileNamedTextIOWrapper, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.fileName = fileName
 
     def __str__(self) -> str:
@@ -110,19 +139,20 @@ class FileNamedTextIOWrapper(io.TextIOWrapper):  # provide string IO in memory b
 
 class FileNamedBytesIO(io.BytesIO):  # provide Bytes IO in memory but behave as a fileName string
     def __init__(self, fileName: str, *args: Any, **kwargs: Any) -> None:
-        super(FileNamedBytesIO, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.fileName = fileName
 
     def close(self) -> None:
-        del self.fileName
-        super(FileNamedBytesIO, self).close()
+        if hasattr(self, 'fileName'):
+            del self.fileName
+        super().close()
 
     def __str__(self) -> str:
         return self.fileName
 
-class ArchiveFileIOError(IOError):
+class ArchiveFileIOError(OSError):
     def __init__(self, fileSource: FileSource, errno: int, fileName: str) -> None:
-        super(ArchiveFileIOError, self).__init__(errno,
+        super().__init__(errno,
                                                  _("Archive {}").format(fileSource.url),
                                                  fileName)
         self.fileName = fileName
@@ -139,16 +169,19 @@ class FileSource:
     url: str | list[str] | None
     basefile: str | list[str] | None
     xfdDocument: etree._ElementTree | None
+    taxonomyPackage: dict[str, str | dict[str, str]] | None
+    mappedPaths: dict[str, str] | None
 
     def __init__(self, url: str, cntlr: Cntlr | None = None, checkIfXmlIsEis: bool = False) -> None:
         self.url = str(url)  # allow either string or FileNamedStringIO
         self.baseIsHttp = isHttpUrl(self.url)
         self.cntlr = cntlr
-        self.type = self.url.lower()[-7:]
-        self.isTarGz = self.type == ".tar.gz"
-        if not self.isTarGz:
-            self.type = self.type[3:]
-        self.isZip = self.type == ".zip"
+        self.isTarGz = self.url.lower().endswith(".tar.gz")
+        if self.isTarGz:
+            self.type = ".tar.gz"
+        else:
+            self.type = os.path.splitext(self.url.lower())[1]
+        self.isZip = self.type == ".zip" or isReportPackageExtension(self.url)
         self.isZipBackslashed = False # windows style backslashed paths
         self.isEis = self.type == ".eis"
         self.isXfd = (self.type == ".xfd" or self.type == ".frm")
@@ -161,6 +194,27 @@ class FileSource:
         self.referencedFileSources = {}  # archive file name, fileSource object
         self.taxonomyPackage = None # taxonomy package
         self.mappedPaths = None  # remappings of path segments may be loaded by taxonomyPackage manifest
+        self.isValid = True # filesource is assumed to be valid until a call to open fails.
+        if not self.isZip:
+            # Try to detect zip files with unrecognized file extensions.
+            try:
+                if self.cntlr is not None and hasattr(self.cntlr, "modelManager"):
+                    basefile = self.cntlr.webCache.getfilename( # cache remapping
+                        self.cntlr.modelManager.disclosureSystem.mappedUrl(self.url)) # local remapping
+                else:
+                    basefile = self.url
+                if basefile:
+                    with openFileStream(self.cntlr, basefile, 'rb') as fileStream:
+                        self.isZip = zipfile.is_zipfile(fileStream)
+            except OSError:
+                # Can't load self.url content. It's not a zip file.
+                # We don't use os.path.isfile because self.url may be an embeded zip file.
+                pass
+            except Exception as err:
+                # Log the error at info level (which is sent to the GUI log), but don't record a validation error.
+                # Validation is deferred to the validation classes. Filesource is unaware of the specific errors that should be raised.
+                self.logError(err)
+
 
         # for SEC xml files, check if it's an EIS anyway
         if (not (self.isZip or self.isEis or self.isXfd or self.isRss) and
@@ -170,14 +224,16 @@ class FileSource:
             elif checkIfXmlIsEis:
                 try:
                     assert self.cntlr is not None
-                    _filename = self.cntlr.webCache.getfilename(self.url)
+                    _filename = self.cntlr.webCache.getfilename(
+                        self.cntlr.modelManager.disclosureSystem.mappedUrl(self.url))
                     assert _filename is not None
-                    file = open(_filename, 'r', errors='replace')
+                    file = open(_filename, errors='replace')
                     l = file.read(256) # may have comments before first element
                     file.close()
-                    if re.match(r"\s*(<[?]xml[^?]+[?]>)?\s*(<!--.*-->\s*)*<(cor[a-z]*:|sdf:)?edgarSubmission", l):
+                    if re.match(r"\s*(<[?]xml[^?]+[?]>)?\s*(<!--.*-->\s*)*<(cor[a-z]*:|sdf:|\w+:)?edgarSubmission", l):
                         self.isEis = True
-                except EnvironmentError as err:
+                except OSError as err:
+                    self.isValid = False
                     if self.cntlr:
                         self.cntlr.addToLog(_("[{0}] {1}").format(type(err).__name__, err))
                     pass
@@ -187,10 +243,11 @@ class FileSource:
             self.cntlr.addToLog(_("[{0}] {1}").format(type(err).__name__, err))
 
     def open(self, reloadCache: bool = False) -> None:
-        if not self.isOpen:
+        if self.isValid and not self.isOpen:
             if (self.isZip or self.isTarGz or self.isEis or self.isXfd or self.isRss or self.isInstalledTaxonomyPackage) and self.cntlr:
                 assert isinstance(self.url, str)
-                self.basefile = self.cntlr.webCache.getfilename(self.url, reload=reloadCache)
+                self.basefile = self.cntlr.webCache.getfilename(
+                    self.cntlr.modelManager.disclosureSystem.mappedUrl(self.url), reload=reloadCache)
             else:
                 self.basefile = self.url
             self.baseurl = self.url # url gets changed by selection
@@ -202,7 +259,8 @@ class FileSource:
                     fileStream = openFileStream(self.cntlr, self.basefile, 'rb')
                     self.fs = zipfile.ZipFile(fileStream, mode="r")
                     self.isOpen = True
-                except EnvironmentError as err:
+                except (OSError, zipfile.BadZipFile) as err:
+                    self.isValid = False
                     self.logError(err)
                     pass
             elif self.isTarGz:
@@ -210,7 +268,8 @@ class FileSource:
                     assert isinstance(self.basefile, str)
                     self.fs = tarfile.open(self.basefile, "r:gz")
                     self.isOpen = True
-                except EnvironmentError as err:
+                except OSError as err:
+                    self.isValid = False
                     self.logError(err)
                     pass
             elif self.isEis:
@@ -220,8 +279,7 @@ class FileSource:
                     assert isinstance(self.basefile, str)
                     file: io.BufferedReader | io.BytesIO | io.StringIO | None = open(self.basefile, 'rb')
                     assert isinstance(file, (io.BufferedReader, io.BytesIO))
-                    more = True
-                    while more:
+                    while True:
                         l = file.read(8)
                         if len(l) < 8:
                             break
@@ -233,7 +291,8 @@ class FileSource:
                             break
                         buf += zlib.decompress(compressedBytes)
                     file.close()
-                except EnvironmentError as err:
+                except OSError as err:
+                    self.isValid = False
                     self.logError(err)
                     pass
                 #uncomment to save for debugging
@@ -252,10 +311,8 @@ class FileSource:
                         self.eisDocument = etree.parse(_file, parser=parser)
                         _file.close()
                         self.isOpen = True
-                    except EnvironmentError as err:
-                        self.logError(err)
-                        return # provide error message later
-                    except etree.LxmlError as err:
+                    except (OSError, etree.LxmlError) as err:
+                        self.isValid = False
                         self.logError(err)
                         return # provide error message later
 
@@ -290,7 +347,8 @@ class FileSource:
                                     ungzippedBytes += readBytes
                                     if len(readBytes) == 0 or (lenUncomp - lenRead) <= 0:
                                         break
-                        except IOError as err:
+                        except OSError:
+                            self.isValid = False
                             pass # provide error message later
 
                         i += lenCompr + 4
@@ -307,10 +365,8 @@ class FileSource:
                     self.xfdDocument = etree.parse(file)
                     file.close()
                     self.isOpen = True
-                except EnvironmentError as err:
-                    self.logError(err)
-                    return # provide error message later
-                except etree.LxmlError as err:
+                except (OSError, etree.LxmlError) as err:
+                    self.isValid = False
                     self.logError(err)
                     return # provide error message later
 
@@ -319,10 +375,8 @@ class FileSource:
                     assert isinstance(self.basefile, str)
                     self.rssDocument = etree.parse(self.basefile)
                     self.isOpen = True
-                except EnvironmentError as err:
-                    self.logError(err)
-                    return # provide error message later
-                except etree.LxmlError as err:
+                except (OSError, etree.LxmlError) as err:
+                    self.isValid = False
                     self.logError(err)
                     return # provide error message later
 
@@ -332,18 +386,18 @@ class FileSource:
                 self.loadTaxonomyPackageMappings()
 
     def loadTaxonomyPackageMappings(self, errors: list[str] = [], expectTaxonomyPackage: bool = False) -> None:
-        if not self.mappedPaths and (self.taxonomyPackageMetadataFiles or expectTaxonomyPackage):
+        if not self.mappedPaths and (self.taxonomyPackageMetadataFiles or expectTaxonomyPackage) and self.cntlr:
             if PackageManager.validateTaxonomyPackage(self.cntlr, self, errors=errors):
                 assert isinstance(self.baseurl, str)
                 metadata = self.baseurl + os.sep + self.taxonomyPackageMetadataFiles[0]
-                self.taxonomyPackage = PackageManager.parsePackage(self.cntlr, self, metadata,  # type: ignore[no-untyped-call]
+                self.taxonomyPackage = PackageManager.parsePackage(self.cntlr, self, metadata,
                                                                    os.sep.join(os.path.split(metadata)[:-1]) + os.sep,
                                                                    errors=errors)
 
                 assert self.taxonomyPackage is not None
-                self.mappedPaths = self.taxonomyPackage.get("remappings")
+                self.mappedPaths = cast('dict[str, str]', self.taxonomyPackage.get("remappings"))
 
-    def openZipStream(self, sourceZipStream: str) -> None:
+    def openZipStream(self, sourceZipStream: BinaryIO) -> None:
         if not self.isOpen:
             assert isinstance(self.url, str)
             self.basefile = self.url
@@ -397,6 +451,19 @@ class FileSource:
     def isTaxonomyPackage(self) -> bool:
         return bool(self.isZip and self.taxonomyPackageMetadataFiles) or self.isInstalledTaxonomyPackage
 
+    @property
+    def isReportPackage(self) -> bool:
+        return self.reportPackage is not None
+
+    @property
+    def reportPackage(self) -> ReportPackage | None:
+        try:
+            self._reportPackage: ReportPackage | None
+            return cast(Optional[ReportPackage], self._reportPackage)
+        except AttributeError:
+            self._reportPackage = ReportPackage.fromFileSource(self)
+            return self._reportPackage
+
     @property
     def taxonomyPackageMetadataFiles(self) -> list[str]:
         for f in (self.dir or []):
@@ -418,8 +485,11 @@ class FileSource:
 
     def isMappedUrl(self, url: str) -> bool:
         if self.mappedPaths is not None:
-            return any(url.startswith(mapFrom)
-                       for mapFrom in self.mappedPaths)
+            if any(url.startswith(mapFrom)
+                       for mapFrom in self.mappedPaths):
+                return True
+        if self.cntlr and self.cntlr.modelManager.disclosureSystem.isMappedUrl(url):
+            return True
         return False
 
     def mappedUrl(self, url: str) -> str:
@@ -428,6 +498,8 @@ class FileSource:
                 if url.startswith(mapFrom):
                     url = mapTo + url[len(mapFrom):]
                     break
+        if self.cntlr:
+            return self.cntlr.modelManager.disclosureSystem.mappedUrl(url)
         return url
 
     def fileSourceContainingFilepath(self, filepath: str | None) -> FileSource | None:
@@ -467,11 +539,7 @@ class FileSource:
         binary: bool = False,
         stripDeclaration: bool = False,
         encoding: str | None = None,
-    ) -> (
-            tuple[io.BytesIO | IO[Any]]
-            | tuple[FileNamedTextIOWrapper, str | None]
-            | tuple[io.TextIOWrapper, str | None]
-        ):
+    ) -> tuple[io.BytesIO | IO[Any]] | tuple[TextIO, str | None]:
         '''
             for text, return a tuple of (open file handle, encoding)
             for binary, return a tuple of (open file handle, )
@@ -499,11 +567,11 @@ class FileSource:
                     if encoding is None:
                         encoding = XmlUtil.encoding(b)
                     if stripDeclaration:
-                        b = stripDeclarationBytes(b)
+                        b = stripDeclarationBytes(b, encoding)
                     return (FileNamedTextIOWrapper(filepath, io.BytesIO(b), encoding=encoding),
                             encoding)
-                except KeyError:
-                    raise ArchiveFileIOError(self, errno.ENOENT, archiveFileName)
+                except KeyError as err:
+                    raise ArchiveFileIOError(self, errno.ENOENT, archiveFileName) from err
             elif archiveFileSource.isTarGz:
                 try:
                     assert isinstance(archiveFileSource.fs, tarfile.TarFile)
@@ -516,16 +584,16 @@ class FileSource:
                     if encoding is None:
                         encoding = XmlUtil.encoding(b)
                     if stripDeclaration:
-                        b = stripDeclarationBytes(b)
+                        b = stripDeclarationBytes(b, encoding)
                     return (FileNamedTextIOWrapper(filepath, io.BytesIO(b), encoding=encoding),
                             encoding)
-                except KeyError:
+                except KeyError as err:
                     # Note 2022-09-06
                     # The following error is raised by mypy, indicating there's a bug here:
                     # Missing positional argument "fileName"
                     # Not fixing this bug as a part of this PR
                     # Also expecting second argument to be int but is str here
-                    raise ArchiveFileIOError(self, archiveFileName) # type: ignore[call-arg, arg-type]
+                    raise ArchiveFileIOError(self, archiveFileName) from err # type: ignore[call-arg, arg-type]
             elif archiveFileSource.isEis:
                 assert self.eisDocument is not None
                 for docElt in self.eisDocument.iter(tag="{http://www.sec.gov/edgar/common}document"):
@@ -594,6 +662,30 @@ class FileSource:
         else:
             return openXmlFileStream(self.cntlr, filepath, stripDeclaration)
 
+    def getBytesSize(self) -> int | None:
+        """
+        Get the size of the zip file in bytes.
+        :return: Size of the zip file in bytes, or None if not applicable.
+        """
+        if isinstance(self.basefile, str) and os.path.isfile(self.basefile):
+            return os.path.getsize(self.basefile)
+        # ZipFile.fp is a private field, but is currently the simplest way for us to
+        # access the internal stream
+        if isinstance(self.fs, zipfile.ZipFile) and (fp := getattr(self.fs, 'fp')) is not None:
+            stream = cast(IO[Any], fp)
+            stream.seek(0, 2)  # Move to the end of the file
+            return stream.tell()  # Report the current position, which is the size of the file
+        return None
+
+    def getBytesSizeEstimate(self) -> int | None:
+        """
+        Get an estimated size of the zip file in bytes.
+        :return: Estimated size of the zip file in bytes, or None if not applicable.
+        """
+        if not isinstance(self.fs, zipfile.ZipFile):
+            return None
+        return sum(zi.compress_size for zi in self.fs.infolist())
+
     def exists(self, filepath: str) -> bool:
         archiveFileSource = self.fileSourceContainingFilepath(filepath)
         if archiveFileSource is not None:
@@ -627,16 +719,9 @@ class FileSource:
         elif self.filesDir is not None:
             return self.filesDir
         elif self.isZip:
-            files: list[str] = []
-
             assert isinstance(self.fs, zipfile.ZipFile)
-            for zipinfo in self.fs.infolist():
-                f = zipinfo.filename
-                if '\\' in f:
-                    self.isZipBackslashed = True
-                    f = f.replace("\\", "/")
-                files.append(f)
-            self.filesDir = files
+            self.isZipBackslashed = any('\\' in zinfo.orig_filename for zinfo in self.fs.infolist())
+            self.filesDir = self.fs.namelist()
         elif self.isTarGz:
             assert isinstance(self.fs, tarfile.TarFile)
             self.filesDir = self.fs.getnames()
@@ -706,7 +791,7 @@ class FileSource:
                         XmlUtil.text(descendantPubDate),
                         instDoc))
                 self.filesDir = files
-            except (EnvironmentError,
+            except (OSError,
                     etree.LxmlError) as err:
                 pass
         elif self.isInstalledTaxonomyPackage:
@@ -726,7 +811,7 @@ class FileSource:
 
     def basedUrl(self, selection: str) -> str:
         baseurl = getattr(self, "baseurl", None)
-        if not baseurl or isHttpUrl(selection) or os.path.isabs(selection):
+        if not baseurl or isHttpUrl(selection) or isLegacyAbs(selection):
             return selection
         assert isinstance(baseurl, str)
         if self.baseIsHttp or os.sep == '/':
@@ -745,6 +830,14 @@ class FileSource:
             else:
                 self.url = self.basedUrl(selection)
 
+    @property
+    def urlBasename(self) -> list[str] | str | None:
+        if isinstance(self.url, str):
+            return os.path.basename(self.url)
+        if isinstance(self.url, list):
+            return [os.path.basename(url) for url in self.url]
+        return None
+
 def openFileStream(
     cntlr: Cntlr | None, filepath: str, mode: str = "r", encoding: str | None = None
 ) -> io.BytesIO | IO[Any]:
@@ -752,20 +845,20 @@ def openFileStream(
     if PackageManager.isMappedUrl(filepath):  # type: ignore[no-untyped-call]
         filepath = PackageManager.mappedUrl(filepath)  # type: ignore[no-untyped-call]
     elif (
-            isHttpUrl(filepath)
-            and cntlr
+            cntlr
             and hasattr(cntlr, "modelManager")
         ): # may be called early in initialization for PluginManager
-        filepath = cntlr.modelManager.disclosureSystem.mappedUrl(filepath)  # type: ignore[no-untyped-call]
+        filepath = cntlr.modelManager.disclosureSystem.mappedUrl(filepath)
     if archiveFilenameParts(filepath): # file is in an archive
         return openFileSource(filepath, cntlr).file(filepath, binary='b' in mode, encoding=encoding)[0]
     if isHttpUrl(filepath) and cntlr:
-        _cacheFilepath = cntlr.webCache.getfilename(filepath, normalize=True) # normalize is separate step in ModelDocument retrieval, combined here
+        _cacheFilepath = cntlr.webCache.getfilename(
+            cntlr.modelManager.disclosureSystem.mappedUrl(filepath), normalize=True) # normalize is separate step in ModelDocument retrieval, combined here
         if _cacheFilepath is None:
-            raise IOError(_("Unable to open file: {0}.").format(filepath))
+            raise OSError(_("Unable to open file: {0}.").format(filepath))
         filepath = _cacheFilepath
     if not filepath and cntlr:
-        raise IOError(_("Unable to open file: \"{0}\".").format(filepath))
+        raise OSError(_("Unable to open file: \"{0}\".").format(filepath))
     # file path may be server (or memcache) or local file system
     if filepath.startswith(SERVER_WEB_CACHE) and cntlr:
         filestream = None
@@ -798,7 +891,7 @@ def openFileStream(
 
 def openXmlFileStream(
     cntlr: Cntlr | None, filepath: str, stripDeclaration: bool = False
-) -> tuple[io.TextIOWrapper, str]:
+) -> tuple[TextIO, str]:
     # returns tuple: (fileStream, encoding)
     openedFileStream = openFileStream(cntlr, filepath, 'rb')
 
@@ -818,19 +911,17 @@ def openXmlFileStream(
         # allow filepath to close
     # this may not be needed for Mac or Linux, needs confirmation!!!
     if text is None:  # ok to read as utf-8
-        return io.open(filepath, 'rt', encoding=encoding or 'utf-8'), encoding
+        return open(filepath, encoding=encoding or 'utf-8'), encoding
     else:
         if stripDeclaration:
-            # strip XML declaration
-            xmlDeclarationMatch = XmlUtil.xmlDeclarationPattern.match(text)
-            if xmlDeclarationMatch: # remove it for lxml
-                if xmlDeclarationMatch.group(1) is not None:
-                    raise XmlUtil.XmlDeclarationLocationException
-                start,end = xmlDeclarationMatch.span(2)
-                text = text[0:start] + text[end:]
+            text = stripDeclarationText(text)
         return (FileNamedStringIO(filepath, initial_value=text), encoding)
 
-def stripDeclarationBytes(xml: bytes) -> bytes:
+def stripDeclarationBytes(xml: bytes, encoding: str | None) -> bytes:
+    if encoding is not None and not encoding.lower().startswith('utf-8'):
+        text = xml.decode(encoding)
+        text = stripDeclarationText(text)
+        return text.encode(encoding)
     xmlStart = xml[0:120]
     indexOfDeclaration = xmlStart.find(b"<?xml")
     if indexOfDeclaration >= 0:
@@ -839,11 +930,22 @@ def stripDeclarationBytes(xml: bytes) -> bytes:
             return xml[indexOfDeclarationEnd + 2:]
     return xml
 
+
+def stripDeclarationText(text: str) -> str:
+    xmlDeclarationMatch = XmlUtil.xmlDeclarationPattern.match(text)
+    if xmlDeclarationMatch:  # remove it for lxml
+        if xmlDeclarationMatch.group(1) is not None:
+            raise XmlUtil.XmlDeclarationLocationException
+        start,end = xmlDeclarationMatch.span(2)
+        text = text[0:start] + text[end:]
+    return text
+
+
 def saveFile(cntlr: Cntlr, filepath: str, contents: str, encoding: str | None = None, mode: str='wt') -> None:
     if isHttpUrl(filepath):
         _cacheFilepath = cntlr.webCache.getfilename(filepath)
         if _cacheFilepath is None:
-            raise IOError(_("Unable to open file: {0}.").format(filepath))
+            raise OSError(_("Unable to open file: {0}.").format(filepath))
         filepath = _cacheFilepath
     # file path may be server (or memcache) or local file system
     if filepath.startswith(SERVER_WEB_CACHE):
@@ -854,7 +956,7 @@ def saveFile(cntlr: Cntlr, filepath: str, contents: str, encoding: str | None =
         _dirpath = os.path.dirname(filepath)
         if not os.path.exists(_dirpath): # directory must exist before io.open
             os.makedirs(_dirpath)
-        with io.open(filepath, mode, encoding=(encoding or 'utf-8')) as f:
+        with open(filepath, mode, encoding=(encoding or 'utf-8')) as f:
             f.write(contents)
 
 # GAE Blobcache