arelle-release 2.17.1__py3-none-any.whl → 2.37.71__py3-none-any.whl

arelle/PackageManager.py

@@ -1,43 +1,59 @@
-'''
-Separated on Jul 28, 2013 from DialogOpenArchive.py
-
+"""
 See COPYRIGHT.md for copyright information.
-'''
+"""
 from __future__ import annotations
-from typing import TYPE_CHECKING
-import sys, os, io, time, json, logging
+
+import json
+import logging
+import os
+import time
 from collections import defaultdict
 from fnmatch import fnmatch
-from lxml import etree
+from typing import TYPE_CHECKING
 from urllib.parse import urljoin
-openFileSource = None
-from arelle import Locale, XmlUtil
+
+from lxml import etree
+
+from arelle import Locale
+from arelle.PythonUtil import isLegacyAbs
+from arelle.packages import PackageValidation
+from arelle.packages.PackageType import PackageType
+from arelle.typing import TypeGetText
 from arelle.UrlUtil import isAbsolute, isHttpUrl
 from arelle.XmlValidate import lxmlResolvingParser
-ArchiveFileIOError = None
-try:
-    from collections import OrderedDict
-except ImportError:
-    OrderedDict = dict # python 3.0 lacks OrderedDict, json file will be in weird order
+
+if TYPE_CHECKING:
+    from arelle.Cntlr import Cntlr
+    from arelle.FileSource import FileSource
 
 TP_XSD = "http://www.xbrl.org/2016/taxonomy-package.xsd"
 CAT_XSD = "http://www.xbrl.org/2016/taxonomy-package-catalog.xsd"
 
-if TYPE_CHECKING:
-    from arelle.Cntlr import Cntlr
+_: TypeGetText
+
+TAXONOMY_PACKAGE_TYPE = PackageType("Taxonomy", "tpe")
 
-EMPTYDICT = {}
+TAXONOMY_PACKAGE_ABORTING_VALIDATIONS = (
+    PackageValidation.validatePackageZipFormat,
+    PackageValidation.validateZipFileSeparators,
+    PackageValidation.validatePackageNotEncrypted,
+    PackageValidation.validateTopLevelFiles,
+    PackageValidation.validateTopLevelDirectories,
+    PackageValidation.validateDuplicateEntries,
+    PackageValidation.validateConflictingEntries,
+    PackageValidation.validateEntries,
+)
+
+TAXONOMY_PACKAGE_NON_ABORTING_VALIDATIONS = (
+    PackageValidation.validateMetadataDirectory,
+)
 
 def baseForElement(element):
     base = ""
     baseElt = element
     while baseElt is not None:
-        baseAttr = baseElt.get("{http://www.w3.org/XML/1998/namespace}base")
-        if baseAttr:
-            if baseAttr.startswith("/"):
-                base = baseAttr
-            else:
-                base = baseAttr + base
+        if baseAttr := baseElt.get("{http://www.w3.org/XML/1998/namespace}base"):
+            base = baseAttr if baseAttr.startswith("/") else baseAttr + base
         baseElt = baseElt.getparent()
     return base
 
@@ -72,10 +88,31 @@ def _parseFile(cntlr, parser, filepath, file, schemaUrl):
     return tree
 
 
-def parsePackage(cntlr, filesource, metadataFile, fileBase, errors=[]):
-    global ArchiveFileIOError
-    if ArchiveFileIOError is None:
-        from arelle.FileSource import ArchiveFileIOError
+def parsePackage(
+    cntlr: Cntlr,
+    filesource: FileSource,
+    metadataFile: str,
+    fileBase: str,
+    errors: list[str] | None = None,
+) -> dict[str, str | dict[str, str]]:
+    if errors is None:
+        errors = []
+    parser = lxmlResolvingParser(cntlr)
+    catalogFile = metadataFile.replace('taxonomyPackage.xml','catalog.xml')
+    remappings = _parseCatalog(cntlr, filesource, parser, catalogFile, fileBase, errors)
+    pkg = _parsePackageMetadata(cntlr, filesource, parser, metadataFile, remappings, errors)
+    return pkg
+
+
+def _parsePackageMetadata(
+    cntlr: Cntlr,
+    filesource: FileSource,
+    parser: etree.XMLParser,
+    metadataFile: str,
+    remappings: dict[str, str],
+    errors: list[str],
+) -> dict[str, str | dict[str, str]]:
+    from arelle.FileSource import ArchiveFileIOError
 
     unNamedCounter = 1
 
@@ -85,16 +122,15 @@ def parsePackage(cntlr, filesource, metadataFile, fileBase, errors=[]):
                    "http://xbrl.org/PR/2015-12-09/taxonomy-package",
                    "http://xbrl.org/2016/taxonomy-package",
                    "http://xbrl.org/WGWD/YYYY-MM-DD/taxonomy-package")
-    catalogNSes = ("urn:oasis:names:tc:entity:xmlns:xml:catalog",)
 
-    pkg = {}
+    pkg = {"remappings": remappings}
 
     currentLang = Locale.getLanguageCode()
-    _file = filesource.file(metadataFile)[0] # URL in zip, plain file in file system or web
     parser = lxmlResolvingParser(cntlr)
     try:
-        tree = _parseFile(cntlr, parser, metadataFile, _file, TP_XSD)
-    except (etree.XMLSyntaxError, etree.DocumentInvalid) as err:
+        metadataFileContent = filesource.file(metadataFile)[0] # URL in zip, plain file in file system or web
+        tree = _parseFile(cntlr, parser, metadataFile, metadataFileContent, TP_XSD)
+    except (etree.XMLSyntaxError, etree.DocumentInvalid, etree.XMLSchemaError) as err:
         cntlr.addToLog(_("Taxonomy package file syntax error %(error)s"),
                        messageArgs={"error": str(err)},
                        messageCode="tpe:invalidMetaDataFile",
@@ -102,10 +138,12 @@ def parsePackage(cntlr, filesource, metadataFile, fileBase, errors=[]):
                        level=logging.ERROR)
         errors.append("tpe:invalidMetaDataFile")
         return pkg
+    except ArchiveFileIOError:
+        return pkg
 
     root = tree.getroot()
     ns = root.tag.partition("}")[0][1:]
-    nsPrefix = "{{{}}}".format(ns)
+    nsPrefix = f"{{{ns}}}"
 
     if ns in  txmyPkgNSes:  # package file
         for eltName in ("identifier", "version", "license", "publisher", "publisherURL", "publisherCountry", "publicationDate"):
@@ -118,7 +156,7 @@ def parsePackage(cntlr, filesource, metadataFile, fileBase, errors=[]):
                 break # take first entry if several
         for eltName in ("name", "description"):
             closest = ''
-            closestLen = 0
+            closestLen = -1
             for m in root.iterchildren(tag=nsPrefix + eltName):
                 s = (m.text or "").strip()
                 eltLang = xmlLang(m)
@@ -126,9 +164,10 @@ def parsePackage(cntlr, filesource, metadataFile, fileBase, errors=[]):
                 if l > closestLen:
                     closestLen = l
                     closest = s
-                elif closestLen == 0 and eltLang.startswith("en"):
+                elif closestLen <= 0 and isinstance(eltLang, str) and eltLang.startswith("en"):
                     closest = s   # pick english if nothing better
             if not closest and eltName == "name":  # assign default name when none in taxonomy package
+                assert isinstance(filesource.baseurl, str)
                 closest = os.path.splitext(os.path.basename(filesource.baseurl))[0]
             pkg[eltName] = closest
         for eltName in ("supersededTaxonomyPackages", "versioningReports"):
@@ -136,7 +175,9 @@ def parsePackage(cntlr, filesource, metadataFile, fileBase, errors=[]):
         for m in root.iterchildren(tag=nsPrefix + "supersededTaxonomyPackages"):
             pkg['supersededTaxonomyPackages'] = [
                 r.text.strip()
-                for r in m.iterchildren(tag=nsPrefix + "taxonomyPackageRef")]
+                for r in m.iterchildren(tag=nsPrefix + "taxonomyPackageRef")
+                if isinstance(r.text, str)
+            ]
         for m in root.iterchildren(tag=nsPrefix + "versioningReports"):
             pkg['versioningReports'] = [
                 r.get("href")
@@ -174,66 +215,12 @@ def parsePackage(cntlr, filesource, metadataFile, fileBase, errors=[]):
         pkg["description"] = "oasis catalog"
         pkg["version"] = "(none)"
 
-    remappings = {}
-    rewriteTree = tree
-    catalogFile = metadataFile
-    if ns in ("http://xbrl.org/PWD/2015-01-14/taxonomy-package",
-              "http://xbrl.org/PR/2015-12-09/taxonomy-package",
-              "http://xbrl.org/WGWD/YYYY-MM-DD/taxonomy-package",
-              "http://xbrl.org/2016/taxonomy-package",
-              "http://xbrl.org/REC/2016-04-19/taxonomy-package"):
-        catalogFile = metadataFile.replace('taxonomyPackage.xml','catalog.xml')
-        try:
-            _file = filesource.file(catalogFile)[0]
-            rewriteTree = _parseFile(cntlr, parser, catalogFile, _file, CAT_XSD)
-        except (etree.XMLSyntaxError, etree.DocumentInvalid) as err:
-            cntlr.addToLog(_("Catalog file syntax error %(error)s"),
-                           messageArgs={"error": str(err)},
-                           messageCode="tpe:invalidCatalogFile",
-                           file=os.path.basename(metadataFile),
-                           level=logging.ERROR)
-            errors.append("tpe:invalidCatalogFile")
-        except ArchiveFileIOError:
-            pass
-    for tag, prefixAttr, replaceAttr in (
-         (nsPrefix + "remapping", "prefix", "replaceWith"), # taxonomy package
-         ("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteSystem", "systemIdStartString", "rewritePrefix"),
-         ("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteURI", "uriStartString", "rewritePrefix")): # oasis catalog
-        for m in rewriteTree.iter(tag=tag):
-            prefixValue = m.get(prefixAttr)
-            replaceValue = m.get(replaceAttr)
-            if prefixValue and replaceValue is not None:
-                if prefixValue not in remappings:
-                    base = baseForElement(m)
-                    if base:
-                        replaceValue = os.path.join(base, replaceValue)
-                    if replaceValue: # neither None nor ''
-                        if not isAbsolute(replaceValue):
-                            if not os.path.isabs(replaceValue):
-                                replaceValue = fileBase + replaceValue
-                            if not isHttpUrl(replaceValue):
-                                replaceValue = replaceValue.replace("/", os.sep)
-                    _normedValue = cntlr.webCache.normalizeUrl(replaceValue)
-                    if replaceValue.endswith(os.sep) and not _normedValue.endswith(os.sep):
-                        _normedValue += os.sep
-                    remappings[prefixValue] = _normedValue
-                else:
-                    cntlr.addToLog(_("Package catalog duplicate rewrite start string %(rewriteStartString)s"),
-                                   messageArgs={"rewriteStartString": prefixValue},
-                                   messageCode="tpe:multipleRewriteURIsForStartString",
-                                   file=os.path.basename(catalogFile),
-                                   level=logging.ERROR)
-                    errors.append("tpe:multipleRewriteURIsForStartString")
-
-
-    pkg["remappings"] = remappings
-
     entryPoints = defaultdict(list)
     pkg["entryPoints"] = entryPoints
 
     for entryPointSpec in tree.iter(tag=nsPrefix + "entryPoint"):
         name = None
-        closestLen = 0
+        closestLen = -1
 
         # find closest match name node given xml:lang match to current language or no xml:lang
         for nameNode in entryPointSpec.iter(tag=nsPrefix + "name"):
@@ -243,28 +230,23 @@ def parsePackage(cntlr, filesource, metadataFile, fileBase, errors=[]):
             if l > closestLen:
                 closestLen = l
                 name = s
-            elif closestLen == 0 and nameLang.startswith("en"):
+            elif closestLen <= 0 and isinstance(nameLang, str) and nameLang.startswith("en"):
                 name = s   # pick english if nothing better
 
         if not name:
             name = _("<unnamed {0}>").format(unNamedCounter)
             unNamedCounter += 1
 
-        epDocCount = 0
         for epDoc in entryPointSpec.iterchildren(nsPrefix + "entryPointDocument"):
             epUrl = epDoc.get('href')
             base = epDoc.get('{http://www.w3.org/XML/1998/namespace}base') # cope with xml:base
-            if base:
-                resolvedUrl = urljoin(base, epUrl)
-            else:
-                resolvedUrl = epUrl
-
-            epDocCount += 1
+            resolvedUrl = urljoin(base, epUrl) if base else epUrl
 
             #perform prefix remappings
             remappedUrl = resolvedUrl
             longestPrefix = 0
             for mapFrom, mapTo in remappings.items():
+                assert isinstance(remappedUrl, str)
                 if remappedUrl.startswith(mapFrom):
                     prefixLength = len(mapFrom)
                     if prefixLength > longestPrefix:
@@ -279,7 +261,7 @@ def parsePackage(cntlr, filesource, metadataFile, fileBase, errors=[]):
 
             # find closest language description
             closest = ''
-            closestLen = 0
+            closestLen = -1
             for m in entryPointSpec.iterchildren(tag=nsPrefix + "description"):
                 s = (m.text or "").strip()
                 eltLang = xmlLang(m)
@@ -287,7 +269,7 @@ def parsePackage(cntlr, filesource, metadataFile, fileBase, errors=[]):
                 if l > closestLen:
                     closestLen = l
                     closest = s
-                elif closestLen == 0 and eltLang.startswith("en"):
+                elif closestLen <= 0 and isinstance(eltLang, str) and eltLang.startswith("en"):
                     closest = s   # pick english if nothing better
             if not closest and name:  # assign default name when none in taxonomy package
                 closest = name
@@ -295,6 +277,63 @@ def parsePackage(cntlr, filesource, metadataFile, fileBase, errors=[]):
 
     return pkg
 
+
+def _parseCatalog(
+    cntlr: Cntlr,
+    filesource: FileSource,
+    parser: etree.XMLParser,
+    catalogFile: str,
+    fileBase: str,
+    errors: list[str],
+) -> dict[str, str]:
+    from arelle.FileSource import ArchiveFileIOError
+    remappings = {}
+    rewriteTree = None
+    try:
+        _file = filesource.file(catalogFile)[0]
+        rewriteTree = _parseFile(cntlr, parser, catalogFile, _file, CAT_XSD)
+    except (etree.XMLSyntaxError, etree.DocumentInvalid) as err:
+        cntlr.addToLog(_("Catalog file syntax error %(error)s"),
+                        messageArgs={"error": str(err)},
+                        messageCode="tpe:invalidCatalogFile",
+                        file=os.path.basename(catalogFile),
+                        level=logging.ERROR)
+        errors.append("tpe:invalidCatalogFile")
+    except ArchiveFileIOError:
+        pass
+    if rewriteTree is not None:
+        for tag, prefixAttr, replaceAttr in (
+            ("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteSystem", "systemIdStartString", "rewritePrefix"),
+            ("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteURI", "uriStartString", "rewritePrefix")): # oasis catalog
+            for m in rewriteTree.iter(tag=tag):
+                prefixValue = m.get(prefixAttr)
+                replaceValue = m.get(replaceAttr)
+                if prefixValue and replaceValue is not None:
+                    if prefixValue not in remappings:
+                        base = baseForElement(m)
+                        if base:
+                            replaceValue = os.path.join(base, replaceValue)
+                        if replaceValue and not isAbsolute(replaceValue):
+                            # neither None nor ''
+                            if not isLegacyAbs(replaceValue):
+                                replaceValue = fileBase + replaceValue
+                            if not isHttpUrl(replaceValue):
+                                replaceValue = replaceValue.replace("/", os.sep)
+                        _normedValue = cntlr.webCache.normalizeUrl(replaceValue)
+                        if replaceValue.endswith(os.sep) and not _normedValue.endswith(os.sep):
+                            _normedValue += os.sep
+                        remappings[prefixValue] = _normedValue
+                    else:
+                        cntlr.addToLog(_("Package catalog duplicate rewrite start string %(rewriteStartString)s"),
+                                    messageArgs={"rewriteStartString": prefixValue},
+                                    messageCode="tpe:multipleRewriteURIsForStartString",
+                                    file=os.path.basename(catalogFile),
+                                    level=logging.ERROR)
+                        errors.append("tpe:multipleRewriteURIsForStartString")
+
+    return remappings
+
+
 # taxonomy package manager
 # plugin control is static to correspond to statically loaded modules
 packagesJsonFile = None
@@ -304,11 +343,11 @@ packagesMappings = {}
 _cntlr = None
 
 def init(cntlr: Cntlr, loadPackagesConfig: bool = True) -> None:
-    global packagesJsonFile, packagesConfig, packagesMappings, _cntlr
+    global packagesJsonFile, packagesConfig, _cntlr
     if loadPackagesConfig:
         try:
             packagesJsonFile = cntlr.userAppDir + os.sep + "taxonomyPackages.json"
-            with io.open(packagesJsonFile, 'rt', encoding='utf-8') as f:
+            with open(packagesJsonFile, encoding='utf-8') as f:
                 packagesConfig = json.load(f)
             packagesConfigChanged = False
         except Exception:
@@ -322,13 +361,15 @@ def init(cntlr: Cntlr, loadPackagesConfig: bool = True) -> None:
     pluginMethodsForClasses = {} # dict by class of list of ordered callable function objects
     _cntlr = cntlr
 
-def reset():  # force reloading modules and plugin infos
-    packagesConfig.clear()  # dict of loaded module pluginInfo objects by module names
-    packagesMappings.clear() # dict by class of list of ordered callable function objects
+def reset() -> None:  # force reloading modules and plugin infos
+    if packagesConfig:
+        packagesConfig.clear()  # dict of loaded module pluginInfo objects by module names
+    if packagesMappings:
+        packagesMappings.clear() # dict by class of list of ordered callable function objects
 
 def orderedPackagesConfig():
-    return OrderedDict(
-        (('packages', [OrderedDict(sorted(_packageInfo.items(),
+    return dict(
+        (('packages', [dict(sorted(_packageInfo.items(),
                                           key=lambda k: {'name': '01',
                                                          'status': '02',
                                                          'version': '03',
@@ -345,12 +386,12 @@ def orderedPackagesConfig():
                                                          'remappings': '14',
                                                          }.get(k[0],k[0])))
                        for _packageInfo in packagesConfig['packages']]),
-         ('remappings',OrderedDict(sorted(packagesConfig['remappings'].items())))))
+         ('remappings',dict(sorted(packagesConfig['remappings'].items())))))
 
 def save(cntlr: Cntlr) -> None:
     global packagesConfigChanged
     if packagesConfigChanged and cntlr.hasFileSystem:
-        with io.open(packagesJsonFile, 'wt', encoding='utf-8') as f:
+        with open(packagesJsonFile, "w", encoding='utf-8') as f:
             jsonStr = str(json.dumps(orderedPackagesConfig(), ensure_ascii=False, indent=2)) # might not be unicode in 2.7
             f.write(jsonStr)
         packagesConfigChanged = False
@@ -392,69 +433,52 @@ def packageNamesWithNewerFileDates():
             pass
     return names
 
-def validateTaxonomyPackage(cntlr, filesource, packageFiles=[], errors=[]) -> bool:
+def validateTaxonomyPackage(cntlr, filesource, errors=[]) -> bool:
         numErrorsOnEntry = len(errors)
-        _dir = filesource.dir
-        if not _dir:
-            raise IOError(_("Unable to open taxonomy package: {0}.").format(filesource.url))
-        if filesource.isZipBackslashed:
-            # see 4.4.17.1 in https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT
-            cntlr.addToLog(_("Taxonomy package directory uses '\\' as file separator"),
-                           messageCode="tpe:invalidArchiveFormat",
-                           file=os.path.basename(filesource.url),
-                           level=logging.ERROR)
-            errors.append("tpe:invalidArchiveFormat")
-            return False
-        # single top level directory
-        topLevels = set(f.partition('/')[0] for f in _dir)
-        topLevelFiles = set(f for f in topLevels if f in _dir) # have no trailing /, not a directory
-        topLevelDirectories = topLevels - topLevelFiles
-        if topLevelFiles:
-            cntlr.addToLog(_("Taxonomy package contains %(count)s top level file(s):  %(topLevelFiles)s"),
-                           messageArgs={"count": len(topLevelFiles),
-                                        "topLevelFiles": ', '.join(sorted(topLevelFiles))},
-                           messageCode="tpe:invalidDirectoryStructure",
-                           file=os.path.basename(filesource.url),
-                           level=logging.ERROR)
-            errors.append("tpe:invalidDirectoryStructure")
-        if len(topLevelDirectories) != 1:
-            cntlr.addToLog(_("Taxonomy package contains %(count)s top level directories:  %(topLevelDirectories)s"),
-                           messageArgs={"count": len(topLevelDirectories),
-                                        "topLevelDirectories": ', '.join(sorted(topLevelDirectories))},
-                           messageCode="tpe:invalidDirectoryStructure",
-                           file=os.path.basename(filesource.url),
-                           level=logging.ERROR)
-            if not topLevelFiles:
-                errors.append("tpe:invalidDirectoryStructure")
-        if not any('META-INF' in f.split('/')[1:][:1] for f in _dir): # only check child of top level
-            cntlr.addToLog(_("Taxonomy package top-level directory does not contain a subdirectory META-INF"),
-                           messageCode="tpe:metadataDirectoryNotFound",
-                           file=os.path.basename(filesource.baseurl),
-                           level=logging.ERROR)
-            errors.append("tpe:metadataDirectoryNotFound")
-        elif any(f.endswith('/META-INF/taxonomyPackage.xml') for f in _dir):
-            for f in _dir:
-                if f.endswith('/META-INF/taxonomyPackage.xml'):
-                    packageFiles.append(f)
-        else:
+        for validator in TAXONOMY_PACKAGE_ABORTING_VALIDATIONS:
+            if validation := validator(TAXONOMY_PACKAGE_TYPE, filesource):
+                cntlr.addToLog(
+                    validation.msg,
+                    messageCode=validation.codes,
+                    messageArgs=validation.args,
+                    file=filesource.urlBasename,
+                    level=validation.level.name,
+                )
+                errors.append(validation.codes)
+                return False
+        for validator in TAXONOMY_PACKAGE_NON_ABORTING_VALIDATIONS:
+            if validation := validator(TAXONOMY_PACKAGE_TYPE, filesource):
+                cntlr.addToLog(
+                    validation.msg,
+                    messageCode=validation.codes,
+                    messageArgs=validation.args,
+                    file=filesource.urlBasename,
+                    level=validation.level.name,
+                )
+                errors.append(validation.codes)
+
+        _dir = filesource.dir or []
+        if not any(f.endswith('/META-INF/taxonomyPackage.xml') for f in _dir):
+            messageCode = "tpe:metadataFileNotFound"
             cntlr.addToLog(_("Taxonomy package does not contain a metadata file */META-INF/taxonomyPackage.xml"),
-                           messageCode="tpe:metadataFileNotFound",
-                           file=os.path.basename(filesource.url),
+                           messageCode=messageCode,
+                           file=filesource.urlBasename,
                            level=logging.ERROR)
-            errors.append("tpe:metadataFileNotFound")
+            errors.append(messageCode)
         return len(errors) == numErrorsOnEntry
 
+
+def discoverPackageFiles(filesource: FileSource) -> list[str]:
+    return [e for e in filesource.dir or [] if e.endswith("/META-INF/taxonomyPackage.xml")]
+
+
 def packageInfo(cntlr, URL, reload=False, packageManifestName=None, errors=[]):
     #TODO several directories, eg User Application Data
     packageFilename = _cntlr.webCache.getfilename(URL, reload=reload, normalize=True)
     if packageFilename:
-        from arelle.FileSource import TAXONOMY_PACKAGE_FILE_NAMES
+        from arelle.FileSource import TAXONOMY_PACKAGE_FILE_NAMES, archiveFilenameParts, openFileSource
         filesource = None
         try:
-            global openFileSource
-            if openFileSource is None:
-                from arelle.FileSource import openFileSource
-            from arelle.FileSource import archiveFilenameParts
             parts = archiveFilenameParts(packageFilename)
             if parts is not None:
                 sourceFileSource = openFileSource(parts[0], _cntlr)
@@ -468,14 +492,13 @@ def packageInfo(cntlr, URL, reload=False, packageManifestName=None, errors=[]):
                 sourceFileSource.close()
             # allow multiple manifests [[metadata, prefix]...] for multiple catalogs
             packages = []
-            packageFiles = []
             if filesource.isZip:
-                validateTaxonomyPackage(cntlr, filesource, packageFiles, errors)
+                validateTaxonomyPackage(cntlr, filesource, errors)
+                packageFiles = discoverPackageFiles(filesource)
                 if not packageFiles:
                     # look for pre-PWD packages
-                    _dir = filesource.dir
-                    _metaInf = '{}/META-INF/'.format(
-                                os.path.splitext(os.path.basename(packageFilename))[0])
+                    _dir = filesource.dir or []
+                    _metaInf = f'{os.path.splitext(os.path.basename(packageFilename))[0]}/META-INF/'
                     if packageManifestName:
                         # pre-pwd
                         packageFiles = [fileName
@@ -488,7 +511,7 @@ def packageInfo(cntlr, URL, reload=False, packageManifestName=None, errors=[]):
                         # root-level META-INF taxonomy packages
                         packageFiles = ['META-INF/taxonomyPackage.xml']
                 if len(packageFiles) < 1:
-                    raise IOError(_("Taxonomy package contained no metadata file: {0}.")
+                    raise OSError(_("Taxonomy package contained no metadata file: {0}.")
                                   .format(', '.join(packageFiles)))
                 # if current package files found, remove any nonconforming package files
                 if any(pf.startswith('_metaInf') for pf in packageFiles) and any(not pf.startswith(_metaInf) for pf in packageFiles):
@@ -497,10 +520,12 @@ def packageInfo(cntlr, URL, reload=False, packageManifestName=None, errors=[]):
                     packageFiles = [pf for pf in packageFiles if pf.startswith('META-INF/')]
 
                 for packageFile in packageFiles:
+                    assert isinstance(filesource.url, str)
                     packageFileUrl = filesource.url + os.sep + packageFile
                     packageFilePrefix = os.sep.join(os.path.split(packageFile)[:-1])
                     if packageFilePrefix:
                         packageFilePrefix += os.sep
+                    assert isinstance(filesource.baseurl, str)
                     packageFilePrefix = filesource.baseurl + os.sep +  packageFilePrefix
                     packages.append([packageFileUrl, packageFilePrefix, packageFile])
             else:
@@ -509,6 +534,7 @@ def packageInfo(cntlr, URL, reload=False, packageManifestName=None, errors=[]):
                                file=os.path.basename(packageFilename),
                                level=logging.ERROR)
                 errors.append("tpe:invalidArchiveFormat")
+                assert isinstance(filesource.url, str)
                 if (os.path.basename(filesource.url) in TAXONOMY_PACKAGE_FILE_NAMES or # individual manifest file
                       (os.path.basename(filesource.url) == "taxonomyPackage.xml" and
                        os.path.basename(os.path.dirname(filesource.url)) == "META-INF")):
@@ -518,7 +544,7 @@ def packageInfo(cntlr, URL, reload=False, packageManifestName=None, errors=[]):
                         packageFilePrefix += os.sep
                     packages.append([packageFileUrl, packageFilePrefix, ""])
                 else:
-                    raise IOError(_("File must be a taxonomy package (zip file), catalog file, or manifest (): {0}.")
+                    raise OSError(_("File must be a taxonomy package (zip file), catalog file, or manifest (): {0}.")
                                   .format(packageFilename, ', '.join(TAXONOMY_PACKAGE_FILE_NAMES)))
             remappings = {}
             packageNames = []
@@ -543,6 +569,7 @@ def packageInfo(cntlr, URL, reload=False, packageManifestName=None, errors=[]):
                 return None
             package = {'name': ", ".join(packageNames),
                        'status': 'enabled',
+                       'identifier': parsedPackage.get('identifier'),
                        'version': parsedPackage.get('version'),
                        'license': parsedPackage.get('license'),
                        'fileDate': time.strftime('%Y-%m-%dT%H:%M:%S UTC', fileDateTuple),
@@ -560,7 +587,7 @@ def packageInfo(cntlr, URL, reload=False, packageManifestName=None, errors=[]):
                        }
             filesource.close()
             return package
-        except (EnvironmentError, etree.XMLSyntaxError):
+        except (OSError, etree.XMLSyntaxError):
             pass
         if filesource:
             filesource.close()
@@ -600,12 +627,12 @@ def rebuildRemappings(cntlr):
 def isMappedUrl(url):
     return (packagesConfig is not None and url is not None and
             any(url.startswith(mapFrom) and not url.startswith(mapTo) # prevent recursion in mapping for url hosted Packages
-                for mapFrom, mapTo in packagesConfig.get('remappings', EMPTYDICT).items()))
+                for mapFrom, mapTo in packagesConfig.get('remappings', {}).items()))
 
 def mappedUrl(url):
     if packagesConfig is not None and url is not None:
         longestPrefix = 0
-        for mapFrom, mapTo in packagesConfig.get('remappings', EMPTYDICT).items():
+        for mapFrom, mapTo in packagesConfig.get('remappings', {}).items():
             if url.startswith(mapFrom):
                 if url.startswith(mapTo):
                     return url # recursive mapping, this is already mapped
@@ -619,13 +646,12 @@ def mappedUrl(url):
 
 def addPackage(cntlr, url, packageManifestName=None):
     newPackageInfo = packageInfo(cntlr, url, packageManifestName=packageManifestName)
-    if newPackageInfo and newPackageInfo.get("name"):
-        name = newPackageInfo.get("name")
-        version = newPackageInfo.get("version")
+    if newPackageInfo and newPackageInfo.get("identifier"):
+        identifier = newPackageInfo.get("identifier")
         j = -1
         packagesList = packagesConfig["packages"]
         for i, _packageInfo in enumerate(packagesList):
-            if _packageInfo['name'] == name and _packageInfo['version'] == version:
+            if _packageInfo['identifier'] == identifier:
                 j = i
                 break
         if 0 <= j < len(packagesList): # replace entry