arelle-release 2.37.46__py3-none-any.whl → 2.38.0__py3-none-any.whl

arelle/plugin/validate/EDINET/ContextRequirement.py

@@ -0,0 +1,58 @@
+"""
+See COPYRIGHT.md for copyright information.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Literal
+
+
+@dataclass(frozen=True)
+class ContextRequirement:
+    """
+    If {elementExists} is set, and {elementDoesNotExist} is not set,
+    any context with ID starting with {contextId} must have {element} value
+    matching the {elementMatch} DEI value (adjusted by {dayAdjustment} days).
+    """
+    contextId: str
+    element: Literal['endDate', 'instant', 'startDate']
+    elementExists: str | None
+    elementDoesNotExist: str | None
+    elementMatch: str
+    dayAdjustment: int = 0  # days to adjust by (e.g. -1 for "day before")
+
+
+# See section 3-4-2 in the Validation Guidelines.
+CONTEXT_REQUIREMENTS = [
+    ContextRequirement('CurrentYearInstant', 'instant', None, None, 'CurrentFiscalYearEndDateDEI'),
+    ContextRequirement('Prior1YearInstant', 'instant', 'EndDateOfQuarterlyOrSemiAnnualPeriodOfNextFiscalYearDEI', None, 'CurrentFiscalYearEndDateDEI'),
+    ContextRequirement('Prior1YearInstant', 'instant', None, 'EndDateOfQuarterlyOrSemiAnnualPeriodOfNextFiscalYearDEI', 'PreviousFiscalYearEndDateDEI'),
+    ContextRequirement('Prior2YearInstant', 'instant', 'EndDateOfQuarterlyOrSemiAnnualPeriodOfNextFiscalYearDEI', None, 'PreviousFiscalYearEndDateDEI'),
+    ContextRequirement('Prior2YearInstant', 'instant', None, 'EndDateOfQuarterlyOrSemiAnnualPeriodOfNextFiscalYearDEI', 'PreviousFiscalYearStartDateDEI', -1),
+    ContextRequirement('CurrentQuarterInstant', 'instant', 'EndDateOfQuarterlyOrSemiAnnualPeriodOfNextFiscalYearDEI', None, 'EndDateOfQuarterlyOrSemiAnnualPeriodOfNextFiscalYearDEI'),
+    ContextRequirement('CurrentQuarterInstant', 'instant', None, 'EndDateOfQuarterlyOrSemiAnnualPeriodOfNextFiscalYearDEI', 'CurrentPeriodEndDateDEI'),
+    ContextRequirement('Prior1QuarterInstant', 'instant', None, 'EndDateOfQuarterlyOrSemiAnnualPeriodOfNextFiscalYearDEI', 'ComparativePeriodEndDateDEI'),
+    ContextRequirement('InterimInstant', 'instant', 'EndDateOfQuarterlyOrSemiAnnualPeriodOfNextFiscalYearDEI', None, 'EndDateOfQuarterlyOrSemiAnnualPeriodOfNextFiscalYearDEI'),
+    ContextRequirement('InterimInstant', 'instant', None, 'EndDateOfQuarterlyOrSemiAnnualPeriodOfNextFiscalYearDEI', 'CurrentPeriodEndDateDEI'),
+    ContextRequirement('Prior1InterimInstant', 'instant', None, 'EndDateOfQuarterlyOrSemiAnnualPeriodOfNextFiscalYearDEI', 'ComparativePeriodEndDateDEI'),
+    ContextRequirement('CurrentYearDuration', 'startDate', None, None, 'CurrentFiscalYearStartDateDEI'),
+    ContextRequirement('Prior1YearDuration', 'startDate', 'NextFiscalYearStartDateDEI', None, 'CurrentFiscalYearStartDateDEI'),
+    ContextRequirement('Prior1YearDuration', 'startDate', None, 'NextFiscalYearStartDateDEI', 'PreviousFiscalYearStartDateDEI'),
+    ContextRequirement('Prior2YearDuration', 'startDate', 'NextFiscalYearStartDateDEI', None, 'PreviousFiscalYearStartDateDEI'),
+    ContextRequirement('CurrentYTDDuration', 'startDate', 'NextFiscalYearStartDateDEI', None, 'NextFiscalYearStartDateDEI'),
+    ContextRequirement('CurrentYTDDuration', 'startDate', None, 'NextFiscalYearStartDateDEI', 'CurrentFiscalYearStartDateDEI'),
+    ContextRequirement('Prior1YTDDuration', 'startDate', None, 'NextFiscalYearStartDateDEI', 'PreviousFiscalYearStartDateDEI'),
+    ContextRequirement('InterimDuration', 'startDate', 'NextFiscalYearStartDateDEI', None, 'NextFiscalYearStartDateDEI'),
+    ContextRequirement('InterimDuration', 'startDate', None, 'NextFiscalYearStartDateDEI', 'CurrentFiscalYearStartDateDEI'),
+    ContextRequirement('Prior1InterimDuration', 'startDate', None, 'NextFiscalYearStartDateDEI', 'PreviousFiscalYearStartDateDEI'),
+    ContextRequirement('CurrentYearDuration', 'endDate', None, None, 'CurrentFiscalYearEndDateDEI'),
+    ContextRequirement('Prior1YearDuration', 'endDate', 'EndDateOfQuarterlyOrSemiAnnualPeriodOfNextFiscalYearDEI', None, 'CurrentFiscalYearEndDateDEI'),
+    ContextRequirement('Prior1YearDuration', 'endDate', None, 'EndDateOfQuarterlyOrSemiAnnualPeriodOfNextFiscalYearDEI', 'PreviousFiscalYearEndDateDEI'),
+    ContextRequirement('Prior2YearDuration', 'endDate', 'EndDateOfQuarterlyOrSemiAnnualPeriodOfNextFiscalYearDEI', None, 'PreviousFiscalYearEndDateDEI'),
+    ContextRequirement('CurrentYTDDuration', 'endDate', 'EndDateOfQuarterlyOrSemiAnnualPeriodOfNextFiscalYearDEI', None, 'EndDateOfQuarterlyOrSemiAnnualPeriodOfNextFiscalYearDEI'),
+    ContextRequirement('CurrentYTDDuration', 'endDate', None, 'EndDateOfQuarterlyOrSemiAnnualPeriodOfNextFiscalYearDEI', 'CurrentPeriodEndDateDEI'),
+    ContextRequirement('Prior1YTDDuration', 'endDate', None, 'EndDateOfQuarterlyOrSemiAnnualPeriodOfNextFiscalYearDEI', 'ComparativePeriodEndDateDEI'),
+    ContextRequirement('InterimDuration', 'endDate', 'EndDateOfQuarterlyOrSemiAnnualPeriodOfNextFiscalYearDEI', None, 'EndDateOfQuarterlyOrSemiAnnualPeriodOfNextFiscalYearDEI'),
+    ContextRequirement('InterimDuration', 'endDate', None, 'EndDateOfQuarterlyOrSemiAnnualPeriodOfNextFiscalYearDEI', 'CurrentPeriodEndDateDEI'),
+    ContextRequirement('Prior1InterimDuration', 'endDate', None, 'EndDateOfQuarterlyOrSemiAnnualPeriodOfNextFiscalYearDEI', 'ComparativePeriodEndDateDEI'),
+]

arelle/plugin/validate/EDINET/ControllerPluginData.py

@@ -3,12 +3,28 @@ See COPYRIGHT.md for copyright information.
 """
 from __future__ import annotations
 
+import zipfile
+from collections import defaultdict
 from dataclasses import dataclass
+from functools import lru_cache
+from pathlib import Path
 from typing import TYPE_CHECKING
 
 from arelle.Cntlr import Cntlr
+from arelle.FileSource import FileSource
+from arelle.ModelValue import QName, TypeXValue
+from arelle.ModelXbrl import ModelXbrl
+from arelle.XmlValidateConst import VALID
 from arelle.typing import TypeGetText
 from arelle.utils.PluginData import PluginData
+from . import Constants
+from .CoverItemRequirements import CoverItemRequirements
+from .DeiRequirements import DeiRequirements, DEI_LOCAL_NAMES
+from .FilingFormat import DocumentType, FilingFormat
+from .NamespaceConfig import NamespaceConfig
+from .ReportFolderType import ReportFolderType
+from .TableOfContentsBuilder import TableOfContentsBuilder
+from .UploadContents import UploadContents, UploadPathInfo
 
 if TYPE_CHECKING:
     from .ManifestInstance import ManifestInstance
@@ -18,11 +34,22 @@ _: TypeGetText
 
 @dataclass
 class ControllerPluginData(PluginData):
+    _deiValues: dict[str, TypeXValue]
+    _loadedModelXbrls: list[ModelXbrl]
     _manifestInstancesById: dict[str, ManifestInstance]
+    _tocBuilder: TableOfContentsBuilder
+    _uploadContents: UploadContents | None
+    _usedFilepaths: set[Path]
 
     def __init__(self, name: str):
         super().__init__(name)
+        self.namespaces = NamespaceConfig()
+        self._deiValues = {}
+        self._loadedModelXbrls = []
         self._manifestInstancesById = {}
+        self._tocBuilder = TableOfContentsBuilder()
+        self._usedFilepaths = set()
+        self._uploadContents = None
 
     def __hash__(self) -> int:
         return id(self)
@@ -33,19 +60,173 @@ class ControllerPluginData(PluginData):
         """
         self._manifestInstancesById[manifestInstance.id] = manifestInstance
 
+    def addModelXbrl(self, modelXbrl: ModelXbrl) -> None:
+        self._loadedModelXbrls.append(modelXbrl)
+        self.setDeiValues(modelXbrl)
+
+    @lru_cache(1)
+    def getCoverItemRequirements(self, jsonPath: Path) -> CoverItemRequirements:
+        return CoverItemRequirements(jsonPath)
+
+    @lru_cache(1)
+    def getDeiRequirements(self, csvPath: Path, deiItems: tuple[QName, ...], filingFormats: tuple[FilingFormat, ...]) -> DeiRequirements:
+        return DeiRequirements(csvPath, deiItems, filingFormats)
+
+    def getDeiValue(self, localName: str) -> TypeXValue:
+        return self._deiValues.get(localName)
+
     def getManifestInstances(self) -> list[ManifestInstance]:
         """
         Retrieve all loaded manifest instances.
         """
         return list(self._manifestInstancesById.values())
 
-    def matchManifestInstance(self, ixdsDocUrls: list[str]) -> ManifestInstance | None:
+    def getTableOfContentsBuilder(self) -> TableOfContentsBuilder:
+        return self._tocBuilder
+
+    def getUploadContents(self) -> UploadContents | None:
+        return self._uploadContents
+
+    def setUploadContents(self, fileSource: FileSource) -> UploadContents:
+        reports = defaultdict(list)
+        uploadPaths = {}
+        for path, zipPath in self.getUploadFilepaths(fileSource).items():
+            if len(path.parts) == 0:
+                continue
+            assert isinstance(fileSource.basefile, str)
+            fullPath = Path(fileSource.basefile) / path
+            parents = list(reversed([p.name for p in path.parents if len(p.name) > 0]))
+            reportFolderType = None
+            isCorrection = True
+            isDirectory = zipPath.is_dir()
+            isInSubdirectory = False
+            reportPath = None
+            if len(parents) > 0:
+                isCorrection = parents[0] != 'XBRL'
+                if not isCorrection:
+                    if len(parents) > 1:
+                        formName = parents[1]
+                        isInSubdirectory = len(parents) > 2
+                        reportFolderType = ReportFolderType.parse(formName)
+                if reportFolderType is None:
+                    formName = parents[0]
+                    isInSubdirectory = len(parents) > 1
+                    reportFolderType = ReportFolderType.parse(formName)
+                if reportFolderType is not None:
+                    reportPath = Path(reportFolderType.value) if isCorrection else Path("XBRL") / reportFolderType.value
+                    if not isCorrection:
+                        reports[reportFolderType].append(path)
+            uploadPaths[path] = UploadPathInfo(
+                fullPath=fullPath,
+                isAttachment=reportFolderType is not None and reportFolderType.isAttachment,
+                isCorrection=isCorrection,
+                isCoverPage=not isDirectory and path.stem.startswith(Constants.COVER_PAGE_FILENAME_PREFIX),
+                isDirectory=isDirectory,
+                isRoot=len(path.parts) == 1,
+                isSubdirectory=isInSubdirectory or (isDirectory and reportFolderType is not None),
+                path=path,
+                reportFolderType=reportFolderType,
+                reportPath=reportPath,
+            )
+        self._uploadContents = UploadContents(
+            reports={k: frozenset(v) for k, v in reports.items() if len(v) > 0},
+            uploadPaths=list(uploadPaths.values())
+        )
+        return self._uploadContents
+
+    @lru_cache(1)
+    def getUploadFilepaths(self, fileSource: FileSource) -> dict[Path, zipfile.Path]:
+        if not self.isUpload(fileSource):
+            return {}
+        paths = {}
+        assert isinstance(fileSource.fs, zipfile.ZipFile)
+        # First, fill in paths from zip file list
+        for file in fileSource.fs.filelist:
+            zipPath = zipfile.Path(fileSource.fs, file.filename)
+            paths[Path(file.filename)] = zipPath
+        # Then, fill in any parent directories that weren't in file list
+        for path in list(paths):
+            for parent in path.parents:
+                if parent in paths:
+                    continue
+                paths[parent] = zipfile.Path(fileSource.fs, parent.as_posix() + '/')
+        return {
+            path: paths[path]
+            for path in sorted(paths)
+        }
+
+    @lru_cache(1)
+    def getUploadFileSizes(self, fileSource: FileSource) -> dict[Path, int]:
+        """
+        Get the sizes of files in the upload directory.
+        :param fileSource: The FileSource instance to get file sizes for.
+        :return: A dictionary mapping file paths to their sizes.
+        """
+        if not self.isUpload(fileSource):
+            return {}
+        assert isinstance(fileSource.fs, zipfile.ZipFile)
+        return {
+            Path(i.filename): i.file_size
+            for i in fileSource.fs.infolist()
+            if not i.is_dir()
+        }
+
+    def getUsedFilepaths(self) -> frozenset[Path]:
+        return frozenset(self._usedFilepaths)
+
+    def isConsolidated(self) -> bool | None:
+        """
+        Is this a consolidated (not individual) filing?
+        Looks for the DEI fact 'WhetherConsolidatedFinancialStatementsArePreparedDEI'
+        within PublicDoc instances. If an explicit True/False value is found, it is returned.
+        If no non-nil value exists, None is returned, which indicates a not applicable state.
+        :return:
+        """
+        for modelXbrl in self.loadedModelXbrls:
+            manifestInstance = self.getManifestInstance(modelXbrl)
+            if manifestInstance is None:
+                continue
+            if manifestInstance.type != ReportFolderType.PUBLIC_DOC.value:
+                continue
+            facts = modelXbrl.factsByLocalName.get('WhetherConsolidatedFinancialStatementsArePreparedDEI', set())
+            for fact in facts:
+                if fact.xValue == True:
+                    return True
+                if fact.xValue == False:
+                    return False
+        return None
+
+    @lru_cache(1)
+    def isUpload(self, fileSource: FileSource) -> bool:
+        fileSource.open()  # Make sure file source is open
+        if (fileSource.fs is None or
+                not isinstance(fileSource.fs, zipfile.ZipFile)):
+            if fileSource.cntlr is not None:
+                fileSource.cntlr.addToLog(
+                    _("The target file is not a zip file, so upload validation was not performed."),
+                    messageCode="EDINET.uploadNotValidated",
+                    file=str(fileSource.url)
+                )
+            return False
+        return True
+
+    @property
+    def loadedModelXbrls(self) -> list[ModelXbrl]:
+        """
+        TODO: Only necessary because cntlr.modelManager.loadedModelXbrls is not reliable
+            in the current conformance suite runner. Remove when that is fixed/replaced.
+        """
+        return self._loadedModelXbrls
+
+    @lru_cache(1)
+    def getManifestInstance(self, modelXbrl: ModelXbrl) -> ManifestInstance | None:
         """
         Match a manifest instance based on the provided ixdsDocUrls.
         A one-to-one mapping must exist between the model's IXDS document URLs and the manifest instance's IXBRL files.
         :param ixdsDocUrls: A model's list of IXDS document URLs.
         :return: A matching ManifestInstance if found, otherwise None.
         """
+        ixdsDocUrls = modelXbrl.ixdsDocUrls
         modelUrls = set(ixdsDocUrls)
         matchedInstance = None
         for instance in self._manifestInstancesById.values():
@@ -70,6 +251,44 @@ class ControllerPluginData(PluginData):
             break
         return matchedInstance
 
+    def setDeiValue(self, localName: str, value: TypeXValue) -> None:
+        if localName in self._deiValues:
+            # Duplicate DEI values will be caught by validations.
+            return
+        self._deiValues[localName] = value
+
+    def setDeiValues(self, modelXbrl: ModelXbrl) -> None:
+        """
+        Set DEI values from the provided modelXbrl.
+        Some EDINET validations rely on both DEI values defined in one instance
+        and other values in a separate instance, so we collect DEI values from
+        all instances and collect them at a controller level.
+        :param modelXbrl:
+        :return:
+        """
+        for localName in DEI_LOCAL_NAMES:
+            for fact in modelXbrl.factsByLocalName.get(localName, ()):
+                if fact.isNil or fact.xValid < VALID:
+                    continue
+                self.setDeiValue(localName, fact.xValue)
+
+    def addUsedFilepath(self, path: Path) -> None:
+        self._usedFilepaths.add(path)
+
+    def hasDocumentType(self, documentTypes: set[DocumentType]) -> bool:
+        """
+        Check if any of the loaded instances are one of the specfied document types.
+        """
+        for modelXbrl in self.loadedModelXbrls:
+            manifestInstance = self.getManifestInstance(modelXbrl)
+            if manifestInstance is None:
+                continue
+            if manifestInstance.filingFormat is None:
+                continue
+            if manifestInstance.filingFormat.documentType in documentTypes:
+                return True
+        return False
+
     @staticmethod
     def get(cntlr: Cntlr, name: str) -> ControllerPluginData:
         controllerPluginData = cntlr.getPluginData(name)

arelle/plugin/validate/EDINET/CoverItemRequirements.py

@@ -0,0 +1,42 @@
+"""
+See COPYRIGHT.md for copyright information.
+"""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+
+# Cover item requirements do not appear to be specifically documented anywhere,
+# so we have inferred them from the documentation and sample filings available to us:
+#   Cover item requirements loaded by starting with "Taxonomy Element List" (ESE140114.xlsx),
+#   which appears to give a mapping of ELR URIs to required cover items (Concepts with
+#   "*CoverPage" local names). However, the sequence of the cover items (validated by EC1004E)
+#   does not match the order in samples #10, #11, #21 and #22. In those cases, the sequences
+#   have been updated to match the samples.
+# A note at the bottom of "3-4-2 Cover Page" within "File Specification for EDINET Filing"
+# (ESE140104.pdf) indicates that cover pages are either generated within EDINET's submission
+# UI or manually. The manual process involves downloading a template cover page HTML file, editing it,
+# and then re-uploading it. This suggests that there are cover page template files that may be the most
+# reliable source of truth for cover item requirements, but we have not been able to access these templates.
+class CoverItemRequirements:
+    _jsonPath: Path
+    _data: dict[str, list[str]] | None
+
+    def __init__(self, jsonPath: Path):
+        self._jsonPath = jsonPath
+        self._data = None
+
+    def _load(self) -> dict[str, list[str]]:
+        if self._data is None:
+            with open(self._jsonPath, encoding='utf-8') as f:
+                self._data = json.load(f)
+        return self._data
+
+    def all(self) -> frozenset[str]:
+        data = self._load()
+        return frozenset(v for values in data.values() for v in values)
+
+    def get(self, roleUri: str) -> list[str]:
+        data = self._load()
+        return data.get(roleUri, [])

arelle/plugin/validate/EDINET/DeiRequirements.py

@@ -0,0 +1,118 @@
+"""
+See COPYRIGHT.md for copyright information.
+"""
+from __future__ import annotations
+
+from enum import Enum
+from pathlib import Path
+
+from arelle.ModelValue import QName
+from .FilingFormat import FilingFormat
+
+
+# DEI requirements parsing is designed so that the contents of Attachment #5
+# in "(Appendix) Taxonomy Extension Guideline" (ESE140111.zip), or future versions,
+# can be easily exported to a CSV file where the rows correspond to DEI concepts
+# and the columns correspond to different formats.
+class DeiRequirements:
+    _csvPath: Path
+    _data: dict[QName, dict[FilingFormat, DeiItemStatus | None]] | None
+    _deiItems: tuple[QName, ...]
+    _filingFormats: tuple[FilingFormat, ...]
+
+    def __init__(self, csvPath: Path, deiItems: tuple[QName, ...], filingFormats: tuple[FilingFormat, ...]):
+        self._csvPath = csvPath
+        self._data = None
+        self._deiItems = deiItems
+        self._filingFormats = filingFormats
+
+    def _load(self) -> dict[QName, dict[FilingFormat, DeiItemStatus | None]]:
+        if self._data is None:
+            with open(self._csvPath, encoding='utf-8') as f:
+                data = [
+                    [
+                        DeiItemStatus.parse(cell) for cell in line.strip().split(',')
+                    ]
+                    for line in f.readlines()
+                ]
+                self._data = {}
+                assert len(data) == len(self._deiItems), \
+                    "Unexpected number of rows in DEI requirements CSV."
+                for rowIndex, row in enumerate(data):
+                    assert len(row) == len(self._filingFormats), \
+                        f"Unexpected number of columns in DEI requirements CSV at row {rowIndex}."
+                    deiItem = self._deiItems[rowIndex]
+                    self._data[deiItem] = {}
+                    for colIndex, cell in enumerate(row):
+                        filingFormat = self._filingFormats[colIndex]
+                        self._data[deiItem][filingFormat] = cell
+        return self._data
+
+
+    def get(self, deiItem: QName, filingFormat: FilingFormat) -> DeiItemStatus | None:
+        data = self._load()
+        if deiItem not in data:
+            return None
+        if filingFormat not in data[deiItem]:
+            return None
+        return data[deiItem][filingFormat]
+
+
+class DeiItemStatus(Enum):
+    # The values of the enum correspond to the symbols used in the spreadsheet.
+    PROHIBITED = '×'
+    OPTIONAL = '△'
+    CONDITIONAL = '○'
+    REQUIRED = '◎'
+
+    @classmethod
+    def parse(cls, value: str) -> DeiItemStatus | None:
+        try:
+            return cls(value)
+        except ValueError:
+            return None
+
+# The below values are based on Attachment #5 in "(Appendix) Taxonomy Extension Guideline" (ESE140111.zip).
+# Column D lists the DEI concepts. Rows purely for grouping are omitted.
+# The order is preserved. The index is used to map to other data structures.
+DEI_LOCAL_NAMES = (
+    # Submitter Information (提出者情報)
+    'EDINETCodeDEI',
+    'FundCodeDEI',
+    'SecurityCodeDEI',
+    'FilerNameInJapaneseDEI',
+    'FilerNameInEnglishDEI',
+    'FundNameInJapaneseDEI',
+    'FundNameInEnglishDEI',
+
+    # Document Submission Information (提出書類情報)
+    'CabinetOfficeOrdinanceDEI',
+    'DocumentTypeDEI',
+    'AccountingStandardsDEI',
+    'WhetherConsolidatedFinancialStatementsArePreparedDEI',
+    'IndustryCodeWhenConsolidatedFinancialStatementsArePreparedInAccordanceWithIndustrySpecificRegulationsDEI',
+    'IndustryCodeWhenFinancialStatementsArePreparedInAccordanceWithIndustrySpecificRegulationsDEI',
+
+    # Current Fiscal Year (当会計期間)
+    'CurrentFiscalYearStartDateDEI',
+    'CurrentPeriodEndDateDEI',
+    'TypeOfCurrentPeriodDEI',
+    'CurrentFiscalYearEndDateDEI',
+
+    # Previous Fiscal Year (比較対象会計期間)
+    'PreviousFiscalYearStartDateDEI',
+    'ComparativePeriodEndDateDEI',
+    'PreviousFiscalYearEndDateDEI',
+
+    # Next Fiscal Year (次の中間期の会計期間)
+    'NextFiscalYearStartDateDEI',
+    'EndDateOfQuarterlyOrSemiAnnualPeriodOfNextFiscalYearDEI',
+
+    'NumberOfSubmissionDEI',
+    'AmendmentFlagDEI',
+    'IdentificationOfDocumentSubjectToAmendmentDEI',
+
+    # Type of Correction (訂正の種類)
+    'ReportAmendmentFlagDEI',
+    'XBRLAmendmentFlagDEI',
+)