arelle-release 2.17.1__py3-none-any.whl → 2.37.71__py3-none-any.whl

arelle/ValidateFilingText.py

@@ -3,19 +3,25 @@ See COPYRIGHT.md for copyright information.
 '''
 from __future__ import annotations
 #import xml.sax, xml.sax.handler
-from lxml.etree import XML, DTD, SubElement, _ElementTree, _Comment, _ProcessingInstruction, XMLSyntaxError, XMLParser
+from lxml.etree import XML, DTD, SubElement, _ElementTree, _Comment, _ProcessingInstruction, _Entity, XMLSyntaxError, XMLParser
 from dataclasses import dataclass
+from PIL import Image as pilImage
 import os, io, base64
 import regex as re
+import string
+from arelle.PythonUtil import isLegacyAbs
 from arelle.XbrlConst import ixbrlAll, xhtml
 from arelle.XmlUtil import setXmlns, xmlstring, xmlDeclarationPattern, XmlDeclarationLocationException
 from arelle.ModelObject import ModelObject
-from arelle.UrlUtil import decodeBase64DataImage, isHttpUrl, scheme
+from arelle.UrlUtil import decodeBase64DataImage, isExternalUrl, isHttpUrl, scheme
 
 XMLpattern = re.compile(r".*(<|&lt;|&#x3C;|&#60;)[A-Za-z_]+[A-Za-z0-9_:]*[^>]*(/>|>|&gt;|/&gt;).*", re.DOTALL)
 CDATApattern = re.compile(r"<!\[CDATA\[(.+)\]\]")
 #EFM table 5-1 and all &xxx; patterns
-docCheckPattern = re.compile(r"&\w+;|[^0-9A-Za-z`~!@#$%&\*\(\)\.\-+ \[\]\{\}\|\\:;\"'<>,_?/=\t\n\r\f]") # won't match &#nnn;
+allowedCharacters =                  string.digits + string.ascii_letters + R"""`~!@#$%&*().-+ {}[]|\:;"'<>,_?/=""" + '\t\n\r\f'
+disallowedCharactersPattern = re.compile(r"""[^0-9          A-Za-z              `~!@#$%&*(). + {}  | :;"'<>,_?/=       \t\n\r\f  \-\[\]\\]""")
+allowedCharactersTranslationDict = dict.fromkeys(map(ord, allowedCharacters))
+disallowedEntityPattern = re.compile(r'&\w+;') # won't match &#nnn;
 namedEntityPattern = re.compile("&[_A-Za-z\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]"
                                 r"[_\-\.:"
                                 "\xB7A-Za-z0-9\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\u0300-\u036F\u203F-\u2040]*;")
@@ -26,7 +32,7 @@ namedEntityPattern = re.compile("&[_A-Za-z\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02
 #                                "\xB7A-Za-z0-9\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\u0300-\u036F\u203F-\u2040]*;")
 
 inlinePattern = re.compile(r"xmlns:[\w.-]+=['\"]http://www.xbrl.org/2013/inlineXBRL['\"]")
-inlineSelfClosedElementPattern = re.compile(r"<(([\w.-]+:)?(\w+))([^\w/][^<]*)?/>")
+inlineSelfClosedElementPattern = re.compile(r"<(?P<element>([\w.-]+:)?(?P<localName>\w+))([^\w/][^<]*)?/>")
 # The ESEF 2022 conformance suite G2-5-1_2 TC3_invalid depends on optional "/".
 # <img src="data:image;base64,iVBOR...
 imgDataMediaBase64Pattern = re.compile(r"data:image(?:/(?P<mimeSubtype>[^,;]*))?(?P<base64>;base64)?,(?P<data>.*)$", re.S)
@@ -450,14 +456,17 @@ def checkfile(modelXbrl, filepath):
             if line == "":
                 break;
             # check for disallowed characters or entity codes
-            for match in docCheckPattern.finditer(line):
+            for match in disallowedEntityPattern.finditer(line):
                 text = match.group()
-                if text.startswith("&"):
-                    if not text in xhtmlEntities:
-                        modelXbrl.error(("EFM.5.02.02.06", "GFM.1.01.02", "FERC.5.02.02.06"),
-                            _("Disallowed entity code %(text)s in file %(file)s line %(line)s column %(column)s"),
-                            modelDocument=filepath, text=text, file=os.path.basename(filepath), line=lineNum, column=match.start())
-                elif validateEntryText and not _isTestcase:
+                if not text in xhtmlEntities:
+                    modelXbrl.error(("EFM.5.02.02.06", "GFM.1.01.02", "FERC.5.02.02.06"),
+                        _("Disallowed entity code %(text)s in file %(file)s line %(line)s column %(column)s"),
+                        modelDocument=filepath, text=text, file=os.path.basename(filepath), line=lineNum, column=match.start())
+            # Finding disallowed characters with a large negated character class can be fairly slow on long lines.
+            # Only run it if there are disallowed characters to find.
+            if validateEntryText and not _isTestcase and line.translate(allowedCharactersTranslationDict):
+                for match in disallowedCharactersPattern.finditer(line):
+                    text = match.group()
                     if len(text) == 1:
                         modelXbrl.error(("EFM.5.02.01.01", "FERC.5.02.01.01"),
                             _("Disallowed character '%(text)s' (%(unicodeIndex)s) in file %(file)s at line %(line)s col %(column)s"),
@@ -487,13 +496,13 @@ def checkfile(modelXbrl, filepath):
             if mayBeInline and inlinePattern.search(line):
                 mayBeInline = False
                 isInline = True
-            if isInline:
+            if isInline and '/>' in line:
                 for match in inlineSelfClosedElementPattern.finditer(line):
-                    selfClosedLocalName = match.group(3)
+                    selfClosedLocalName = match.group('localName')
                     if selfClosedLocalName not in elementsWithNoContent:
                         modelXbrl.warning("ixbrl:selfClosedTagWarning",
                                           _("Self-closed element \"%(element)s\" may contain text or other elements and should not use self-closing tag syntax (/>) when empty; change these to end-tags in file %(file)s line %(line)s column %(column)s"),
-                                          modelDocument=filepath, element=match.group(1), file=os.path.basename(filepath), line=lineNum, column=match.start())
+                                          modelDocument=filepath, element=match.group('element'), file=os.path.basename(filepath), line=lineNum, column=match.start())
             result.append(line)
             lineNum += 1
     result = ''.join(result)
@@ -595,7 +604,7 @@ def validateTextBlockFacts(modelXbrl):
                         eltTag = elt.tag
                         if isinstance(elt, ModelObject) and elt.namespaceURI == xhtml:
                             eltTag = elt.localName
-                        elif isinstance(elt, (_ElementTree, _Comment, _ProcessingInstruction)):
+                        elif isinstance(elt, (_ElementTree, _Comment, _ProcessingInstruction, _Entity)):
                             continue # comment or other non-parsed element
                         else:
                             eltTag = elt.tag
@@ -611,7 +620,7 @@ def validateTextBlockFacts(modelXbrl):
                                 if attrTag in efmBlockedInlineHtmlElementAttributes.get(eltTag,()):
                                     modelXbrl.error(("EFM.5.02.05.disallowedAttribute", "FERC.5.02.05.disallowedAttribute"),
                                         _("%(validatedObjectLabel)s has disallowed attribute on element <%(element)s>: %(attribute)s=\"%(value)s\""),
-                                        modelObject=elt, validatedObjectLabel=validatedObjectLabel,
+                                        modelObject=elt, validatedObjectLabel=f1.qname,
                                         element=eltTag, attribute=attrTag, value=attrValue)
                             if ((attrTag == "href" and eltTag == "a") or
                                 (attrTag == "src" and eltTag == "img")):
@@ -622,7 +631,7 @@ def validateTextBlockFacts(modelXbrl):
                                         attribute=attrTag, element=eltTag)
                                 elif eltTag == "a" and (not allowedExternalHrefPattern or allowedExternalHrefPattern.match(attrValue)):
                                     pass
-                                elif scheme(attrValue) in ("http", "https", "ftp"):
+                                elif isExternalUrl(attrValue):
                                     modelXbrl.error(("EFM.6.05.16.externalReference", "FERC.6.05.16.externalReference"),
                                         _("Fact %(fact)s of context %(contextID)s has an invalid external reference in '%(attribute)s' for <%(element)s>"),
                                         modelObject=f1, fact=f1.qname, contextID=f1.contextID,
@@ -651,10 +660,11 @@ def validateTextBlockFacts(modelXbrl):
                                             attribute=attrValue, element=eltTag)
                                     else:   # test file contents
                                         try:
-                                            if validateGraphicFile(f1, attrValue) != attrValue.lower()[-3:]:
+                                            graphicFileFormat = validateGraphicFile(f1, attrValue, checkIfAnimated=allowedImageTypes.get("disallow-animated"))
+                                            if graphicFileFormat != attrValue.lower()[-3:]:
                                                 modelXbrl.error(("EFM.6.05.16.graphicFileContent", "FERC.6.05.16.graphicFileContent"),
-                                                    _("Fact %(fact)s of context %(contextID)s references a graphics file which has invalid format '%(attribute)s' for <%(element)s>"),
-                                                    modelObject=f1, fact=f1.qname, contextID=f1.contextID,
+                                                    _("Fact %(fact)s of context %(contextID)s references a graphics file which has invalid format %(format)s in '%(attribute)s' for <%(element)s>"),
+                                                    modelObject=f1, fact=f1.qname, contextID=f1.contextID, format=graphicFileFormat,
                                                     attribute=attrValue, element=eltTag)
                                         except IOError as err:
                                             modelXbrl.error(("EFM.6.05.16.graphicFileError", "FERC.6.05.16.graphicFileError"),
@@ -720,7 +730,7 @@ def validateHtmlContent(modelXbrl, referenceElt, htmlEltTree, validatedObjectLab
     for elt in htmlEltTree.iter():
         if isinstance(elt, ModelObject) and elt.namespaceURI == xhtml:
             eltTag = elt.localName
-        elif isinstance(elt, (_ElementTree, _Comment, _ProcessingInstruction)):
+        elif isinstance(elt, (_ElementTree, _Comment, _ProcessingInstruction, _Entity)):
             continue # comment or other non-parsed element
         else:
             eltTag = elt.tag
@@ -763,7 +773,7 @@ def validateHtmlContent(modelXbrl, referenceElt, htmlEltTree, validatedObjectLab
                         messageCodes=("EFM.6.05.34.activeContent", "EFM.5.02.05.activeContent", "FERC.6.05.34.activeContent", "FERC.5.02.05.activeContent"))
                 elif eltTag == "a" and (not allowedExternalHrefPattern or allowedExternalHrefPattern.match(attrValue)):
                     pass
-                elif scheme(attrValue) in ("http", "https", "ftp"):
+                elif isExternalUrl(attrValue):
                     modelXbrl.error(messageCodePrefix + "externalReference",
                         _("%(validatedObjectLabel)s has an invalid external reference in '%(attribute)s' for <%(element)s>: %(value)s"),
                         modelObject=elt, validatedObjectLabel=validatedObjectLabel,
@@ -794,10 +804,11 @@ def validateHtmlContent(modelXbrl, referenceElt, htmlEltTree, validatedObjectLab
                             messageCodes=("EFM.6.05.34.graphicFileType", "EFM.5.02.05.graphicFileType", "FERC.6.05.34.graphicFileType", "FERC.5.02.05.graphicFileType"))
                     else:   # test file contents
                         try:
-                            if validateGraphicFile(referenceElt, attrValue) != attrValue.lower()[-3:]:
+                            graphicFileFormat = validateGraphicFile(referenceElt, attrValue, checkIfAnimated=allowedImageTypes.get("disallow-animated"))
+                            if graphicFileFormat != attrValue.lower()[-3:]:
                                 modelXbrl.error(messageCodePrefix +"graphicFileContent",
-                                    _("%(validatedObjectLabel)s references a graphics file which has invalid format '%(attribute)s' for <%(element)s>"),
-                                    modelObject=elt, validatedObjectLabel=validatedObjectLabel,
+                                    _("%(validatedObjectLabel)s references a graphics file which has invalid format %(format)s in '%(attribute)s' for <%(element)s>"),
+                                    modelObject=elt, validatedObjectLabel=validatedObjectLabel, format=graphicFileFormat,
                                     attribute=attrValue, element=eltTag,
                                     messageCodes=("EFM.6.05.34.graphicFileContent", "EFM.5.02.05.graphicFileContent", "FERC.6.05.34.graphicFileContent", "FERC.5.02.05.graphicFileContent"))
                         except IOError as err:
@@ -968,17 +979,23 @@ def validateGraphicHeaderType(data: bytes) -> str:
     else:
         return "unrecognized"
 
-def validateGraphicFile(elt, graphicFile):
+def validateGraphicFile(elt, graphicFile, checkIfAnimated=False):
     base = elt.modelDocument.baseForElement(elt)
     normalizedUri = elt.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(graphicFile, base)
     if not elt.modelXbrl.fileSource.isInArchive(normalizedUri):
         normalizedUri = elt.modelXbrl.modelManager.cntlr.webCache.getfilename(normalizedUri)
     # all Edgar graphic files must be resolved locally
     #normalizedUri = elt.modelXbrl.modelManager.cntlr.webCache.getfilename(normalizedUri)
+    hdrType = None
     if normalizedUri: # may be None if file doesn't exist
         with elt.modelXbrl.fileSource.file(normalizedUri,binary=True)[0] as fh:
-            return validateGraphicHeaderType(fh.read(11))
-    return None
+            hdrType = validateGraphicHeaderType(fh.read(11))
+            if checkIfAnimated and hdrType == "gif": # check if animated
+                fh.seek(0) # position at start of file
+                with pilImage.open(fh) as img:
+                    if getattr(img, "is_animated", False):
+                        hdrType = "gif-animated"
+    return hdrType
 
 def referencedFiles(modelXbrl, localFilesOnly=True):
     initModelDocumentTypeReferences()
@@ -991,7 +1008,7 @@ def referencedFiles(modelXbrl, localFilesOnly=True):
                 if (attrTag in ("href", "src") and
                     scheme(attrValue) not in ("data", "javascript") and (
                         not localFilesOnly or
-                        (not isHttpUrl(attrValue) and not os.path.isabs(attrValue)))):
+                        (not isHttpUrl(attrValue) and not isLegacyAbs(attrValue)))):
                     attrValue = attrValue.partition('#')[0].strip() # remove anchor
                     if attrValue not in ("", "."): # ignore anchor references to base document
                         base = docElt.modelDocument.baseForElement(docElt)

arelle/ValidateInfoset.py

@@ -2,8 +2,10 @@
 See COPYRIGHT.md for copyright information.
 '''
 from collections import defaultdict
+import io, os
+from lxml import etree
 from arelle.ModelDocument import Type
-from arelle.ModelValue import qname
+from arelle.ModelValue import qname, dateTime
 from arelle import XmlUtil, XbrlConst
 from arelle.ValidateXbrlCalcs import inferredPrecision, inferredDecimals
 
@@ -191,8 +193,82 @@ def resolvePath(modelXbrl, namespaceId):
             return doc.idObjects[id]
     return None
 
+def stripTime(periodAspect):
+    a = [t for t in periodAspect if t]
+    p = []
+    if len(a) > 1:
+        p.append(dateTime(a[0].replace("Z","")))
+    if len(a) > 0:
+        p.append(dateTime(a[-1].replace("Z",""), addOneDay=True))
+    return tuple(p)
+
+def compareRenderingInfosetElts(modelXbrl, sourceElt, comparisonElt):
+    sourceEltTag = sourceElt.tag if sourceElt is not None else '(no more elements)'
+    comparisonEltTag = comparisonElt.tag if comparisonElt is not None else '(no more elements)'
+    if sourceEltTag != comparisonEltTag:
+        modelXbrl.error("arelle:tableModelElementMismatch",
+            _("Table layout model expecting %(elt1)s found %(elt2)s source path %(elt1path)s comparison line %(elt2line)s path %(elt2path)s, srcElt %(elt1xml)s cmpElt %(elt2xml)s"),
+            modelObject=modelXbrl, elt1=sourceEltTag, elt2=comparisonEltTag,
+            elt1path=sourceElt.getroottree().getpath(sourceElt), elt2path=comparisonElt.getroottree().getpath(comparisonElt),
+            elt2line=comparisonElt.sourceline,
+            elt1xml=etree.tostring(sourceElt),elt2xml=etree.tostring(comparisonElt))
+    elif sourceEltTag == "{http://xbrl.org/2014/table/model}cell":
+        ceSrcIter = sourceElt.iter("{http://xbrl.org/2014/table/model}fact",
+                                   "{http://xbrl.org/2014/table/model}label")
+        ceCmpIter = comparisonElt.iter("{http://xbrl.org/2014/table/model}fact",
+                                       "{http://xbrl.org/2014/table/model}label")
+        ceSrcElt = next(ceSrcIter, None)
+        ceCmpElt = next(ceCmpIter, None)
+        while (ceSrcElt is not None and ceCmpElt is not None):
+            compareRenderingInfosetElts(modelXbrl, ceSrcElt, ceCmpElt)
+            ceSrcElt = next(ceSrcIter, None)
+            ceCmpElt = next(ceCmpIter, None)
+        srcConstraints = {}
+        cmpConstraints = {}
+        for srcE, cstrts in ((sourceElt, srcConstraints),(comparisonElt, cmpConstraints)):
+            for e in srcE.iter("{http://xbrl.org/2014/table/model}constraint"):
+                cstrtTag = e.get("tag","")
+                cstrtAspect = e.findtext("{http://xbrl.org/2014/table/model}aspect")
+                if cstrtAspect:
+                    if e.find("{http://xbrl.org/2014/table/model}value") is None:
+                        cstrts[cstrtAspect,cstrtTag] = ()
+                    else:
+                        cstrts[cstrtAspect,cstrtTag] = tuple((f.text or "").strip() for f in e.find("{http://xbrl.org/2014/table/model}value") for f in e.iter())
+        for c in (srcConstraints, cmpConstraints):
+            for k, v in c.items():
+                if isinstance(k, tuple) and k[0] == "period":
+                    # remove end dates time parts if zero (because not consistently reported in conf suite expected outputs)
+                    c[k] = stripTime(c[k])
+        if srcConstraints != cmpConstraints:
+            modelXbrl.error("arelle:tableModelConstraintsMismatch",
+                _("Table layout model constraints %(src)s expecting %(cmp)s source path %(elt1path)s comparison line %(elt2line)s path %(elt2path)s"),
+                modelObject=modelXbrl, src=",".join(str(s) for s in sorted(srcConstraints.items())), cmp=",".join(str(s) for s in sorted(cmpConstraints.items())),
+                elt1path=sourceElt.getroottree().getpath(sourceElt), elt2path=comparisonElt.getroottree().getpath(comparisonElt),
+                elt2line=comparisonElt.sourceline)
+    else:
+        text1 = (sourceElt.text or '').strip() or '(none)'
+        text2 = (comparisonElt.text or '').strip() or '(none)'
+        if text1 != text2:
+            modelXbrl.error("arelle:tableModelTextMismatch",
+                _("Table layout model comparison element %(elt)s expecting text %(text1)s found %(text2)s source line %(elt1line)s comparison line %(elt2line)s"),
+                modelObject=modelXbrl, elt=sourceElt.tag, text1=text1, text2=text2,
+                elt1line=sourceElt.sourceline, elt2line=comparisonElt.sourceline)
+        attrs1 = dict(sourceElt.items())
+        attrs2 = dict(comparisonElt.items())
+        # remove attributes not to be compared
+        for attr in ("{http://www.w3.org/XML/1998/namespace}base",
+                     ):
+            if attr in attrs1: del attrs1[attr]
+            if attr in attrs2: del attrs2[attr]
+        if attrs1 != attrs2:
+            modelXbrl.error("arelle:tableModelAttributesMismatch",
+                _("Table layout model comparison element %(elt)s expecting attributes %(attrs1)s found %(attrs2)s source line %(elt1line)s comparison line %(elt2line)s"),
+                modelObject=modelXbrl, elt=sourceElt.tag,
+                attrs1=', '.join('{0}="{1}"'.format(k,v) for k,v in sorted(attrs1.items())),
+                attrs2=', '.join('{0}="{1}"'.format(k,v) for k,v in sorted(attrs2.items())),
+                elt1line=sourceElt.sourceline, elt2line=comparisonElt.sourceline)
+
 def validateRenderingInfoset(modelXbrl, comparisonFile, sourceDoc):
-    from lxml import etree
     try:
         # etree.parse does not successfully parse files within archives
         if modelXbrl.fileSource.isInArchive(comparisonFile):
@@ -200,6 +276,25 @@ def validateRenderingInfoset(modelXbrl, comparisonFile, sourceDoc):
             comparisonDoc = etree.parse(file)
         else:
             comparisonDoc = etree.parse(comparisonFile)
+        numErrorsAtStart = len(modelXbrl.errors)
+        # uncomment to debug when layout model corrupted after creation
+        # with open("/Users/hermf/temp/temp2.xml", "wb") as fh:
+        #     fh.write(etree.tostring(sourceDoc, pretty_print=True))
+        numSrcTblElts = len(sourceDoc.findall(".//{http://xbrl.org/2014/table/model}table"))
+        numCmpTblElts = len(comparisonDoc.findall(".//{http://xbrl.org/2014/table/model}table"))
+        if numSrcTblElts != numCmpTblElts:
+            modelXbrl.error("arelle:tableModelNumberOfTables",
+                _("Table layout model comparison expects %(numCmpTblElts)s table elements, layout has %(numSrcTblElts)s table elements"),
+                modelObject=modelXbrl, numSrcTblElts=numSrcTblElts, numCmpTblElts=numCmpTblElts)
+        if False: #numCmpTblElts > 1 and comparisonFile.endswith("linkrole-parameter.xml"):
+            # reverse tables for this test case
+            tblElts = sourceDoc.findall(".//{http://xbrl.org/2014/table/model}table")
+            tblElts[0].addprevious(tblElts[1])
+        # order comparison document headers in z - y - x order
+        headerElts = comparisonDoc.findall(".//{http://xbrl.org/2014/table/model}headers")
+        if len(headerElts) == 3:
+            if headerElts[0].get("axis") == "y" and headerElts[2].get("axis") == "z":
+                headerElts[0].addprevious(headerElts[2])
         sourceIter = sourceDoc.iter()
         comparisonIter = comparisonDoc.iter()
         sourceElt = next(sourceIter, None)
@@ -214,38 +309,15 @@ def validateRenderingInfoset(modelXbrl, comparisonFile, sourceDoc):
                 sourceElt = next(sourceIter, None)
             while (isinstance(comparisonElt, etree._Comment)):
                 comparisonElt = next(comparisonIter, None)
-            sourceEltTag = sourceElt.tag if sourceElt is not None else '(no more elements)'
-            comparisonEltTag = comparisonElt.tag if comparisonElt is not None else '(no more elements)'
-            if sourceEltTag != comparisonEltTag:
-                modelXbrl.error("arelle:tableModelElementMismatch",
-                    _("Table model expecting %(elt1)s found %(elt2)s source line %(elt1line)s comparison line %(elt2line)s"),
-                    modelObject=modelXbrl, elt1=sourceEltTag, elt2=comparisonEltTag,
-                    elt1line=sourceElt.sourceline, elt2line=comparisonElt.sourceline)
-            else:
-                text1 = (sourceElt.text or '').strip() or '(none)'
-                text2 = (comparisonElt.text or '').strip() or '(none)'
-                if text1 != text2:
-                    modelXbrl.error("arelle:tableModelTextMismatch",
-                        _("Table model comparison element %(elt)s expecting text %(text1)s found %(text2)s source line %(elt1line)s comparison line %(elt2line)s"),
-                        modelObject=modelXbrl, elt=sourceElt.tag, text1=text1, text2=text2,
-                        elt1line=sourceElt.sourceline, elt2line=comparisonElt.sourceline)
-                attrs1 = dict(sourceElt.items())
-                attrs2 = dict(comparisonElt.items())
-                # remove attributes not to be compared
-                for attr in ("{http://www.w3.org/XML/1998/namespace}base",
-                             ):
-                    if attr in attrs1: del attrs1[attr]
-                    if attr in attrs2: del attrs2[attr]
-                if attrs1 != attrs2:
-                    modelXbrl.error("arelle:tableModelAttributesMismatch",
-                        _("Table model comparison element %(elt)s expecting attributes %(attrs1)s found %(attrs2)s source line %(elt1line)s comparison line %(elt2line)s"),
-                        modelObject=modelXbrl, elt=sourceElt.tag,
-                        attrs1=', '.join('{0}="{1}"'.format(k,v) for k,v in sorted(attrs1.items())),
-                        attrs2=', '.join('{0}="{1}"'.format(k,v) for k,v in sorted(attrs2.items())),
-                        elt1line=sourceElt.sourceline, elt2line=comparisonElt.sourceline)
+            if sourceElt is None or not any(e is not None for e in sourceElt.iterancestors("{http://xbrl.org/2014/table/model}cell")):
+                compareRenderingInfosetElts(modelXbrl, sourceElt, comparisonElt)
             sourceElt = next(sourceIter, None)
             comparisonElt = next(comparisonIter, None)
+        # option to save sourceDoc when errors detected
+        #if len(modelXbrl.errors) > numErrorsAtStart:
+        #    with io.open("/Users/hermf/temp/temp/" + os.path.basename(comparisonFile), "wb") as fh:
+        #        fh.write(etree.tostring(sourceDoc, encoding="utf-8", pretty_print=True))
     except (IOError, etree.LxmlError) as err:
         modelXbrl.error("arelle:tableModelFileError",
-            _("Table model comparison file %(xmlfile)s error %(error)s"),
+            _("Table layout model comparison file %(xmlfile)s error %(error)s"),
             modelObject=modelXbrl, xmlfile=comparisonFile, error=str(err))

arelle/ValidateUtr.py

@@ -1,20 +1,44 @@
 '''
 See COPYRIGHT.md for copyright information.
 '''
+from __future__ import annotations
+
 from lxml import etree
 from arelle import ModelDocument
 from collections import defaultdict
 
+from arelle.ModelValue import QName
 from arelle.ModelXbrl import ModelXbrl
 
 DIVISOR = "*DIV*"
 
 class UtrEntry(): # use slotted class for execution efficiency
-    __slots__ = ("id", "unitId", "nsUnit", "itemType", "nsItemType", "isSimple",
-                 "numeratorItemType", "nsNumeratorItemType",
+    id: str | None
+    unitId: str | None
+    unitName: str | None
+    nsUnit: str | None
+    itemType: str | None
+    nsItemType: str | None
+    numeratorItemType: str | None
+    nsNumeratorItemType: str | None
+    denominatorItemType: str | None
+    nsDenominatorItemType: str | None
+    isSimple: bool
+    symbol: str | None
+    definition: str | None
+    status: str | None
+
+    __slots__ = ("id", "unitId", "unitName", "nsUnit", "itemType", "nsItemType", "isSimple",
+                 "numeratorItemType", "nsNumeratorItemType", "definition",
                  "denominatorItemType", "nsDenominatorItemType", "symbol",
                  "status")
 
+    def qname(self) -> QName | None:
+        if not self.nsUnit:
+            return None
+        prefix = self.nsUnit.split('/')[-1]
+        return QName(prefix=prefix, namespaceURI=self.nsUnit, localName=self.unitId)
+
     def __repr__(self):
         return "utrEntry({})".format(', '.join("{}={}".format(n, getattr(self,n))
                                                for n in self.__slots__))
@@ -52,6 +76,7 @@ def loadUtr(modelXbrl, statusFilters=None): # Build a dictionary of item types t
                 u = UtrEntry()
                 u.id = unitElt.get("id")
                 u.unitId = unitElt.findtext("{http://www.xbrl.org/2009/utr}unitId")
+                u.unitName = unitElt.findtext("{http://www.xbrl.org/2009/utr}unitName")
                 u.nsUnit = (unitElt.findtext("{http://www.xbrl.org/2009/utr}nsUnit") or None) # None if empty entry
                 u.itemType = unitElt.findtext("{http://www.xbrl.org/2009/utr}itemType")
                 u.nsItemType = unitElt.findtext("{http://www.xbrl.org/2009/utr}nsItemType")
@@ -61,6 +86,7 @@ def loadUtr(modelXbrl, statusFilters=None): # Build a dictionary of item types t
                 u.nsDenominatorItemType = unitElt.findtext("{http://www.xbrl.org/2009/utr}nsDenominatorItemType")
                 u.isSimple = all(e is None for e in (u.numeratorItemType, u.nsNumeratorItemType, u.denominatorItemType, u.nsDenominatorItemType))
                 u.symbol = unitElt.findtext("{http://www.xbrl.org/2009/utr}symbol")
+                u.definition = unitElt.findtext("{http://www.xbrl.org/2009/utr}definition")
                 u.status = unitElt.findtext("{http://www.xbrl.org/2009/utr}status")
                 if u.status in statusFilters:
                     # TO DO: This indexing scheme assumes that there are no name clashes in item types of the registry.
@@ -108,17 +134,18 @@ def utrSymbol(modelType, unitMeasures):
     return ValidateUtr(modelType.modelXbrl).utrSymbol(unitMeasures[0], unitMeasures[1])
 
 class ValidateUtr:
+
     def __init__(self, modelXbrl: ModelXbrl, messageLevel: str="ERROR", messageCode: str="utre:error-NumericFactUtrInvalid") -> None:
         self.modelXbrl = modelXbrl
         self.messageLevel = messageLevel
         self.messageCode = messageCode
         if getattr(modelXbrl.modelManager.disclosureSystem, "utrItemTypeEntries", None) is None:
             loadUtr(modelXbrl)
-        self.utrItemTypeEntries = modelXbrl.modelManager.disclosureSystem.utrItemTypeEntries
+        self.utrItemTypeEntries: dict[str, dict[str, UtrEntry]] = modelXbrl.modelManager.disclosureSystem.utrItemTypeEntries
 
     def validateFacts(self):
         modelXbrl = self.modelXbrl
-        if modelXbrl.modelDocument.type in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL):
+        if modelXbrl.modelDocument.type in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL, ModelDocument.Type.INLINEXBRLDOCUMENTSET):
             modelXbrl.modelManager.cntlr.showStatus(_("Validating for Unit Type Registry").format())
             utrInvalidFacts = []
             for f in modelXbrl.facts: