arelle-release 2.37.28__py3-none-any.whl → 2.37.30__py3-none-any.whl

arelle/Locale.py

@@ -238,9 +238,11 @@ def getLocale() -> str | None:
         systemLocale = tryRunCommand("pwsh", "-Command", "Get-Culture | Select -ExpandProperty IetfLanguageTag")
     if pythonCompatibleLocale := findCompatibleLocale(systemLocale):
         _locale = pythonCompatibleLocale
-    elif sys.version_info < (3, 12):
+    elif sys.version_info < (3, 12) or (3, 13, 3) <= sys.version_info[:3] <= (3, 13, 4):
         # Using locale.setlocale(...) because getlocale() in Python versions prior to 3.12 incorrectly aliased C.UTF-8 to en_US.UTF-8.
         # https://github.com/python/cpython/issues/74940
+        # Similar bug was reintroduced in Python 3.13.3 and fixed in 3.13.5.
+        # https://github.com/python/cpython/pull/135347
         _locale = locale.setlocale(locale.LC_CTYPE).partition(POSIX_LOCALE_ENCODING_SEPARATOR)[0]
     else:
         _locale = locale.getlocale()[0]

arelle/ModelTestcaseObject.py

@@ -319,9 +319,15 @@ class ModelTestcaseVariation(ModelObject):
         errorElements = XmlUtil.descendants(self, None, "error")
         resultElement = XmlUtil.descendant(self, None, "result")
         if isinstance(errorElements,list) and len(errorElements) > 0:
-            return [ModelValue.qname(e, e.stringValue) if not e.get("nonStandardErrorCodes")
-                    else e.stringValue
-                    for e in errorElements]
+            errorCodes = []
+            for errorElement in errorElements:
+                if errorElement.get("nonStandardErrorCodes"):
+                    errorCode = errorElement.stringValue
+                else:
+                    errorCode = ModelValue.qname(errorElement, errorElement.stringValue)
+                num = int(errorElement.attr("num") or 1)
+                errorCodes.extend([errorCode] * num)
+            return errorCodes
         #else:
         #    errorElement = errorElements[1]
         #    if errorElement is not None and not errorElement.get("nonStandardErrorCodes"):

arelle/Validate.py

@@ -338,17 +338,18 @@ class Validate:
             loadedModels.append(modelXbrl)
             PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors)
         else: # not a multi-schemaRef versioning report
+            readMeFirstUriIsArchive = isReportPackageExtension(readMeFirstUri)
             readMeFirstUriIsEmbeddedZipFile = False
             if self.useFileSource.isArchive and not isLegacyAbs(readMeFirstUri):
-                if isReportPackageExtension(readMeFirstUri):
+                if readMeFirstUriIsArchive:
                     readMeFirstUriIsEmbeddedZipFile = True
                 else:
                     normalizedReadMeFirstUri = self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri, baseForElement)
                     archivePath = FileSource.archiveFilenameParts(normalizedReadMeFirstUri)
                     if archivePath:
                         with self.useFileSource.fs.open(archivePath[1]) as embeddedFile:
-                            readMeFirstUriIsEmbeddedZipFile = zipfile.is_zipfile(embeddedFile)
-            if not readMeFirstUriIsEmbeddedZipFile:
+                            readMeFirstUriIsArchive = readMeFirstUriIsEmbeddedZipFile = zipfile.is_zipfile(embeddedFile)
+            if not readMeFirstUriIsArchive:
                 modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
                                             readMeFirstUri,
                                             _("validating"),
@@ -400,7 +401,16 @@ class Validate:
                                 # resolve an IXDS in entrypoints
                                 for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ArchiveIxds"):
                                     pluginXbrlMethod(self, filesource,entrypoints)
-                                filesource.select(entrypoints[0].get("file", None))
+                                for entrypoint in entrypoints:
+                                    filesource.select(entrypoint.get("file", None))
+                                    modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
+                                                               filesource,
+                                                               _("validating"),
+                                                               base=filesource.basefile + "/",
+                                                               errorCaptureLevel=errorCaptureLevel,
+                                                               ixdsTarget=modelTestcaseVariation.ixdsTarget,
+                                                               errors=preLoadingErrors)
+                                    loadedModels.append(modelXbrl)
                     except Exception as err:
                         self.modelXbrl.error("exception:" + type(err).__name__,
                             _("Testcase variation validation exception: %(error)s, entry URL: %(instance)s"),
@@ -427,15 +437,16 @@ class Validate:
                         # Legacy ESEF conformance suite logic.
                         for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ReportPackageIxds"):
                                 filesource.select(pluginXbrlMethod(filesource, **_rptPkgIxdsOptions))
-                    modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
-                                                filesource,
-                                                _("validating"),
-                                                base=baseForElement,
-                                                errorCaptureLevel=errorCaptureLevel,
-                                                ixdsTarget=modelTestcaseVariation.ixdsTarget,
-                                                isLoadable=modelTestcaseVariation.variationDiscoversDTS or filesource.url,
-                                                errors=preLoadingErrors)
-                    loadedModels.append(modelXbrl)
+                    if len(loadedModels) == 0:
+                        modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
+                                                    filesource,
+                                                    _("validating"),
+                                                    base=baseForElement,
+                                                    errorCaptureLevel=errorCaptureLevel,
+                                                    ixdsTarget=modelTestcaseVariation.ixdsTarget,
+                                                    isLoadable=modelTestcaseVariation.variationDiscoversDTS or filesource.url,
+                                                    errors=preLoadingErrors)
+                        loadedModels.append(modelXbrl)
 
         for model in loadedModels:
             modelXbrl.isTestcaseVariation = True
@@ -730,7 +741,11 @@ class Validate:
                 indexPath = indexPath[len(baseZipFile) + 1:]
             indexPath = indexPath.replace("\\", "/")
         variationIdPath = f'{indexPath}:{modelTestcaseVariation.id}'
-        if userExpectedErrors := testcaseExpectedErrors.get(variationIdPath):
+        userExpectedErrors = []
+        for userPattern, userErrors in testcaseExpectedErrors.items():
+            if fnmatch.fnmatch(variationIdPath, userPattern):
+                userExpectedErrors.extend(userErrors)
+        if userExpectedErrors:
             if expected is None:
                 expected = []
             if isinstance(expected, str):

arelle/WebCache.py

@@ -27,6 +27,13 @@ try:
     import ssl
 except ImportError:
     ssl = None
+
+try:
+    import truststore
+except ImportError:
+    # truststore requires Python > 3.9
+    truststore = None
+
 from arelle.FileSource import SERVER_WEB_CACHE, archiveFilenameParts
 from arelle.PluginManager import pluginClassMethods
 from arelle.UrlUtil import isHttpUrl
@@ -267,12 +274,10 @@ class WebCache:
             self.http_auth_handler = proxyhandlers.HTTPBasicAuthHandler()
             proxyHandlers = [self.proxy_handler, self.proxy_auth_handler, self.http_auth_handler]
         if ssl:
-            # Attempts to load the default CA certificates from the OS.
-            context = ssl.create_default_context()
+            # Attempt to load the default CA certificates from the OS using truststore if available, else fallback to OpenSSL's default context.
+            context = truststore.SSLContext(ssl.PROTOCOL_TLS_CLIENT) if truststore else ssl.create_default_context()
             # Include certifi certificates (Mozilla’s carefully curated
-            # collection) for systems with outdated certs and for platforms
-            # that we're unable to load certs from (macOS and some Linux
-            # distros.)
+            # collection) for systems with outdated certs.
             context.load_verify_locations(cafile=certifi.where())
             if self.noCertificateCheck:  # this is required in some Akamai environments, such as sec.gov
                 context.check_hostname = False

arelle/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2.37.28'
-__version_tuple__ = version_tuple = (2, 37, 28)
+__version__ = version = '2.37.30'
+__version_tuple__ = version_tuple = (2, 37, 30)

arelle/plugin/validate/EDINET/DisclosureSystems.py

@@ -0,0 +1 @@
+DISCLOSURE_SYSTEM_EDINET = 'EDINET'

arelle/plugin/validate/EDINET/FormType.py

@@ -0,0 +1,105 @@
+"""
+See COPYRIGHT.md for copyright information.
+"""
+from __future__ import annotations
+
+from enum import Enum
+from functools import cached_property, lru_cache
+from pathlib import Path
+
+
+class FormType(Enum):
+    ATTACH_DOC = "AttachDoc"
+    AUDIT_DOC = "AuditDoc"
+    ENGLISH_DOC = "EnglishDoc"
+    PRIVATE_ATTACH = "PrivateAttach"
+    PRIVATE_DOC = "PrivateDoc"
+    PUBLIC_DOC = "PublicDoc"
+
+    @classmethod
+    def parse(cls, value: str) -> FormType | None:
+        try:
+            return cls(value)
+        except ValueError:
+            return None
+
+    @cached_property
+    def extensionCategory(self) -> ExtensionCategory | None:
+        return FORM_TYPE_EXTENSION_CATEGORIES.get(self, None)
+
+    @cached_property
+    def manifestName(self) -> str:
+        return f'manifest_{self.value}.xml'
+
+    @cached_property
+    def manifestPath(self) -> Path:
+        return self.xbrlDirectory / self.manifestName
+
+    @cached_property
+    def xbrlDirectory(self) -> Path:
+        return Path('XBRL') / str(self.value)
+
+    @lru_cache(1)
+    def getValidExtensions(self, isAmendment: bool, isSubdirectory: bool) -> frozenset[str] | None:
+        if self.extensionCategory is None:
+            return None
+        return self.extensionCategory.getValidExtensions(isAmendment, isSubdirectory)
+
+
+class ExtensionCategory(Enum):
+    ATTACH = 'ATTACH'
+    DOC = 'DOC'
+    ENGLISH_DOC = 'ENGLISH_DOC'
+
+    def getValidExtensions(self, isAmendment: bool, isSubdirectory: bool) -> frozenset[str] | None:
+        amendmentMap = VALID_EXTENSIONS[isAmendment]
+        categoryMap = amendmentMap.get(self, None)
+        if categoryMap is None:
+            return None
+        return categoryMap.get(isSubdirectory, None)
+
+
+FORM_TYPE_EXTENSION_CATEGORIES = {
+    FormType.ATTACH_DOC: ExtensionCategory.ATTACH,
+    FormType.AUDIT_DOC: ExtensionCategory.DOC,
+    FormType.ENGLISH_DOC: ExtensionCategory.ENGLISH_DOC,
+    FormType.PRIVATE_ATTACH: ExtensionCategory.ATTACH,
+    FormType.PRIVATE_DOC: ExtensionCategory.DOC,
+    FormType.PUBLIC_DOC: ExtensionCategory.DOC,
+}
+
+
+HTML_EXTENSIONS = frozenset({'.htm', '.html', '.xhtml'})
+IMAGE_EXTENSIONS = frozenset({'.jpeg', '.jpg', '.gif', '.png'})
+ASSET_EXTENSIONS = frozenset(HTML_EXTENSIONS | IMAGE_EXTENSIONS)
+XBRL_EXTENSIONS = frozenset(HTML_EXTENSIONS | {'.xml', '.xsd'})
+ATTACH_EXTENSIONS = frozenset(HTML_EXTENSIONS | {'.pdf', })
+ENGLISH_DOC_EXTENSIONS = frozenset(ASSET_EXTENSIONS | frozenset({'.pdf', '.xml', '.txt'}))
+
+# Is Amendment -> Category -> Is Subdirectory
+VALID_EXTENSIONS = {
+    False: {
+        ExtensionCategory.ATTACH: {
+            False: ATTACH_EXTENSIONS,
+            True: ASSET_EXTENSIONS,
+        },
+        ExtensionCategory.DOC: {
+            False: XBRL_EXTENSIONS,
+            True: ASSET_EXTENSIONS
+        },
+    },
+    True: {
+        ExtensionCategory.ATTACH: {
+            False: ATTACH_EXTENSIONS,
+            True: ASSET_EXTENSIONS,
+        },
+        ExtensionCategory.DOC: {
+            False: HTML_EXTENSIONS,
+            True: ASSET_EXTENSIONS,
+        },
+        ExtensionCategory.ENGLISH_DOC: {
+            False: ENGLISH_DOC_EXTENSIONS,
+            True: ENGLISH_DOC_EXTENSIONS,
+        },
+    },
+}

arelle/plugin/validate/EDINET/PluginValidationDataExtension.py

@@ -0,0 +1,111 @@
+"""
+See COPYRIGHT.md for copyright information.
+"""
+from __future__ import annotations
+
+import zipfile
+from collections import defaultdict
+from dataclasses import dataclass
+from functools import lru_cache
+from pathlib import Path
+
+from arelle.ModelXbrl import ModelXbrl
+from arelle.ValidateXbrl import ValidateXbrl
+from arelle.typing import TypeGetText
+from arelle.utils.PluginData import PluginData
+from .FormType import FormType
+
+_: TypeGetText
+
+
+@dataclass(frozen=True)
+class UploadContents:
+    amendmentPaths: dict[FormType, frozenset[Path]]
+    directories: frozenset[Path]
+    forms: dict[FormType, frozenset[Path]]
+    unknownPaths: frozenset[Path]
+
+
+@dataclass
+class PluginValidationDataExtension(PluginData):
+    _primaryModelXbrl: ModelXbrl | None = None
+
+    # Identity hash for caching.
+    def __hash__(self) -> int:
+        return id(self)
+
+    @lru_cache(1)
+    def shouldValidateUpload(self, val: ValidateXbrl) -> bool:
+        """
+        Determine if the upload validation should be performed on this model.
+
+        Upload validation should not be performed if the target document is
+        not a zipfile.
+
+        Upload validation should only be performed once for the entire package,
+        not duplicated for each model. To facilitate this with Arelle's validation
+        system which largely prevents referencing other models, we can use `--keepOpen`
+        and check if the given model is the first to be loaded.
+        :param val: The ValidateXbrl instance with a model to check.
+        :return: True if upload validation should be performed, False otherwise.
+        """
+        modelXbrl = val.modelXbrl
+        if modelXbrl == val.testModelXbrl:
+            # Not running within a testcase
+            if modelXbrl != modelXbrl.modelManager.loadedModelXbrls[0]:
+                return False
+        if not modelXbrl.fileSource.fs:
+            return False  # No stream
+        if not isinstance(modelXbrl.fileSource.fs, zipfile.ZipFile):
+            return False  # Not a zipfile
+        return True
+
+    @lru_cache(1)
+    def getUploadContents(self, modelXbrl: ModelXbrl) -> UploadContents:
+        uploadFilepaths = self.getUploadFilepaths(modelXbrl)
+        amendmentPaths = defaultdict(list)
+        unknownPaths = []
+        directories = []
+        forms = defaultdict(list)
+        for path in uploadFilepaths:
+            parents = list(reversed([p.name for p in path.parents if len(p.name) > 0]))
+            if len(parents) == 0:
+                continue
+            if parents[0] == 'XBRL':
+                if len(parents) > 1:
+                    formName = parents[1]
+                    formType = FormType.parse(formName)
+                    if formType is not None:
+                        forms[formType].append(path)
+                        continue
+            formName = parents[0]
+            formType = FormType.parse(formName)
+            if formType is not None:
+                amendmentPaths[formType].append(path)
+                continue
+            if len(path.suffix) == 0:
+                directories.append(path)
+                continue
+            unknownPaths.append(path)
+        return UploadContents(
+            amendmentPaths={k: frozenset(v) for k, v in amendmentPaths.items() if len(v) > 0},
+            directories=frozenset(directories),
+            forms={k: frozenset(v) for k, v in forms.items() if len(v) > 0},
+            unknownPaths=frozenset(unknownPaths)
+        )
+
+    @lru_cache(1)
+    def getUploadFilepaths(self, modelXbrl: ModelXbrl) -> list[Path]:
+        if not modelXbrl.fileSource.fs or \
+                not isinstance(modelXbrl.fileSource.fs, zipfile.ZipFile):
+            modelXbrl.warning(
+                codes="EDINET.uploadNotValidated",
+                msg=_("The target file is not a zip file, so upload validation could not be performed.")
+            )
+            return []
+        paths = set()
+        for name in modelXbrl.fileSource.fs.namelist():
+            path = Path(name)
+            paths.add(path)
+            paths.update(path.parents)
+        return sorted(paths)

arelle/plugin/validate/EDINET/ValidationPluginExtension.py

@@ -0,0 +1,29 @@
+"""
+See COPYRIGHT.md for copyright information.
+"""
+from __future__ import annotations
+
+from typing import Any
+
+from arelle.ValidateXbrl import ValidateXbrl
+from arelle.typing import TypeGetText
+from arelle.utils.validate.ValidationPlugin import ValidationPlugin
+from .DisclosureSystems import DISCLOSURE_SYSTEM_EDINET
+from .PluginValidationDataExtension import PluginValidationDataExtension
+
+_: TypeGetText
+
+
+class ValidationPluginExtension(ValidationPlugin):
+    def newPluginData(self, validateXbrl: ValidateXbrl) -> PluginValidationDataExtension:
+        disclosureSystem = validateXbrl.disclosureSystem.name
+        if disclosureSystem == DISCLOSURE_SYSTEM_EDINET:
+            pass
+        else:
+            raise ValueError(f'Invalid EDINET disclosure system: {disclosureSystem}')
+        return PluginValidationDataExtension(
+            self.name,
+        )
+
+    def modelXbrlLoadComplete(self, *args: Any, **kwargs: Any) -> None:
+        return None

arelle/plugin/validate/EDINET/__init__.py

@@ -0,0 +1,119 @@
+"""
+See COPYRIGHT.md for copyright information.
+- [Operation Guides](https://disclosure2dl.edinet-fsa.go.jp/guide/static/disclosure/WEEK0060.html)
+- [Document Search](https://disclosure2.edinet-fsa.go.jp/week0020.aspx)
+"""
+from __future__ import annotations
+
+import zipfile
+from collections import defaultdict
+from pathlib import Path
+from typing import Any
+
+from lxml import etree
+from lxml.etree import _Element
+
+from arelle.FileSource import FileSource
+from arelle.Version import authorLabel, copyrightLabel
+from .ValidationPluginExtension import ValidationPluginExtension
+from .rules import upload
+
+PLUGIN_NAME = "Validate EDINET"
+DISCLOSURE_SYSTEM_VALIDATION_TYPE = "EDINET"
+
+
+validationPlugin = ValidationPluginExtension(
+    name=PLUGIN_NAME,
+    disclosureSystemConfigUrl=Path(__file__).parent / "resources" / "config.xml",
+    validationTypes=[DISCLOSURE_SYSTEM_VALIDATION_TYPE],
+    validationRuleModules=[
+        upload,
+    ],
+)
+
+
+def disclosureSystemTypes(*args: Any, **kwargs: Any) -> tuple[tuple[str, str], ...]:
+    return validationPlugin.disclosureSystemTypes
+
+
+def disclosureSystemConfigURL(*args: Any, **kwargs: Any) -> str:
+    return validationPlugin.disclosureSystemConfigURL
+
+
+def _parseManifestDoc(xmlRootElement: _Element, base: Path) -> dict[str, list[Path]]:
+    sets = defaultdict(list)
+    for instanceElt in xmlRootElement.iter(tag="{http://disclosure.edinet-fsa.go.jp/2013/manifest}instance"):
+        instanceId = str(instanceElt.attrib["id"])
+        for ixbrlElt in instanceElt.iter(tag="{http://disclosure.edinet-fsa.go.jp/2013/manifest}ixbrl"):
+            uri = ixbrlElt.text.strip() if ixbrlElt.text is not None else None
+            if uri:
+                sets[instanceId].append(base / uri)
+    return sets
+
+
+def fileSourceEntrypointFiles(filesource: FileSource, inlineOnly: bool, *args: Any, **kwargs: Any) -> list[dict[str, Any]] | None:
+    manifests = {}
+    if filesource.isArchive:
+        if filesource.isTaxonomyPackage:
+            return None
+        if filesource.reportPackage is not None:
+            return None
+        for _archiveFile in (filesource.dir or ()):
+            if not Path(_archiveFile).stem.startswith('manifest'):
+                continue
+            assert isinstance(filesource.fs, zipfile.ZipFile), \
+                "The EDINET plugin only supports archives in .zip format."
+            with filesource.fs.open(_archiveFile) as manifestDoc:
+                base = Path(_archiveFile).parent
+                xmlRootElement = etree.fromstring(manifestDoc.read())
+                manifests.update(_parseManifestDoc(xmlRootElement, base))
+    elif (dirpath := Path(str(filesource.url))).is_dir():
+        for file in dirpath.rglob("*"):
+            if not file.is_file():
+                continue
+            if not file.stem.startswith('manifest'):
+                continue
+            with open(file, 'rb') as manifestDoc:
+                base = file.parent
+                xmlRootElement = etree.fromstring(manifestDoc.read())
+                manifests.update(_parseManifestDoc(xmlRootElement, base))
+    if len(manifests) == 0:
+        return None
+
+    entrypointFiles = []
+    for instanceId, uris in manifests.items():
+        entrypoints = []
+        for uri in uris:
+            filesource.select(str(uri))
+            entrypoints.append({"file": filesource.url})
+        entrypointFiles.append({'ixds': entrypoints})
+    return entrypointFiles
+
+
+def modelXbrlLoadComplete(*args: Any, **kwargs: Any) -> None:
+    return validationPlugin.modelXbrlLoadComplete(*args, **kwargs)
+
+
+def validateFinally(*args: Any, **kwargs: Any) -> None:
+    return validationPlugin.validateFinally(*args, **kwargs)
+
+
+def validateXbrlFinally(*args: Any, **kwargs: Any) -> None:
+    return validationPlugin.validateXbrlFinally(*args, **kwargs)
+
+
+__pluginInfo__ = {
+    "name": PLUGIN_NAME,
+    "version": "0.0.1",
+    "description": "Validation plugin for the EDINET taxonomies.",
+    "license": "Apache-2",
+    "author": authorLabel,
+    "copyright": copyrightLabel,
+    "import": ("inlineXbrlDocumentSet",),
+    "DisclosureSystem.Types": disclosureSystemTypes,
+    "DisclosureSystem.ConfigURL": disclosureSystemConfigURL,
+    "FileSource.EntrypointFiles": fileSourceEntrypointFiles,
+    "ModelXbrl.LoadComplete": modelXbrlLoadComplete,
+    "Validate.XBRL.Finally": validateXbrlFinally,
+    "ValidateFormula.Finished": validateFinally,
+}

arelle/plugin/validate/EDINET/resources/config.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<DisclosureSystems
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:noNamespaceSchemaLocation="../../../../config/disclosuresystems.xsd">
+    <!-- see arelle/config/disclosuresystems.xml for full comments -->
+    <DisclosureSystem
+            names="EDINET|edinet"
+            description="Checks for EDINET."
+            validationType="EDINET"
+            exclusiveTypesPattern="EFM|GFM|FERC|HMRC|SBR.NL|EBA|EIOPA|ESEF"
+    />
+</DisclosureSystems>