arelle-release 2.37.17__py3-none-any.whl → 2.37.19__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of arelle-release might be problematic. Click here for more details.

Files changed (54) hide show
  1. arelle/CntlrWinMain.py +4 -1
  2. arelle/ModelValue.py +1 -0
  3. arelle/PythonUtil.py +132 -24
  4. arelle/ViewWinTree.py +15 -9
  5. arelle/_version.py +2 -2
  6. arelle/plugin/OimTaxonomy/ModelValueMore.py +15 -0
  7. arelle/plugin/OimTaxonomy/ValidateDTS.py +484 -0
  8. arelle/plugin/OimTaxonomy/ViewXbrlTxmyObj.py +240 -0
  9. arelle/plugin/OimTaxonomy/XbrlAbstract.py +16 -0
  10. arelle/plugin/OimTaxonomy/XbrlConcept.py +67 -0
  11. arelle/plugin/OimTaxonomy/XbrlConst.py +261 -0
  12. arelle/plugin/OimTaxonomy/XbrlCube.py +91 -0
  13. arelle/plugin/OimTaxonomy/XbrlDimension.py +38 -0
  14. arelle/plugin/OimTaxonomy/XbrlDts.py +152 -0
  15. arelle/plugin/OimTaxonomy/XbrlEntity.py +16 -0
  16. arelle/plugin/OimTaxonomy/XbrlGroup.py +22 -0
  17. arelle/plugin/OimTaxonomy/XbrlImportedTaxonomy.py +22 -0
  18. arelle/plugin/OimTaxonomy/XbrlLabel.py +31 -0
  19. arelle/plugin/OimTaxonomy/XbrlNetwork.py +100 -0
  20. arelle/plugin/OimTaxonomy/XbrlProperty.py +28 -0
  21. arelle/plugin/OimTaxonomy/XbrlReference.py +33 -0
  22. arelle/plugin/OimTaxonomy/XbrlReport.py +24 -0
  23. arelle/plugin/OimTaxonomy/XbrlTableTemplate.py +35 -0
  24. arelle/plugin/OimTaxonomy/XbrlTaxonomy.py +93 -0
  25. arelle/plugin/OimTaxonomy/XbrlTaxonomyObject.py +154 -0
  26. arelle/plugin/OimTaxonomy/XbrlTransform.py +17 -0
  27. arelle/plugin/OimTaxonomy/XbrlTypes.py +23 -0
  28. arelle/plugin/OimTaxonomy/XbrlUnit.py +17 -0
  29. arelle/plugin/OimTaxonomy/__init__.py +1038 -0
  30. arelle/plugin/OimTaxonomy/resources/iso4217.json +4479 -0
  31. arelle/plugin/OimTaxonomy/resources/oim-taxonomy-schema.json +935 -0
  32. arelle/plugin/OimTaxonomy/resources/ref.json +333 -0
  33. arelle/plugin/OimTaxonomy/resources/transform-types.json +2481 -0
  34. arelle/plugin/OimTaxonomy/resources/types.json +727 -0
  35. arelle/plugin/OimTaxonomy/resources/utr.json +3046 -0
  36. arelle/plugin/OimTaxonomy/resources/xbrlSpec.json +1082 -0
  37. arelle/plugin/saveOIMTaxonomy.py +311 -0
  38. arelle/plugin/validate/NL/PluginValidationDataExtension.py +36 -2
  39. arelle/plugin/validate/NL/__init__.py +3 -0
  40. arelle/plugin/validate/NL/rules/nl_kvk.py +84 -2
  41. {arelle_release-2.37.17.dist-info → arelle_release-2.37.19.dist-info}/METADATA +2 -1
  42. {arelle_release-2.37.17.dist-info → arelle_release-2.37.19.dist-info}/RECORD +54 -19
  43. tests/integration_tests/validation/README.md +2 -0
  44. tests/integration_tests/validation/conformance_suite_configurations/nl_inline_2024.py +15 -4
  45. tests/integration_tests/validation/run_conformance_suites.py +10 -1
  46. tests/integration_tests/validation/validation_util.py +10 -5
  47. tests/unit_tests/arelle/test_frozen_dict.py +176 -0
  48. tests/unit_tests/arelle/test_frozen_ordered_set.py +315 -0
  49. tests/unit_tests/arelle/test_import.py +31 -0
  50. tests/unit_tests/arelle/test_ordered_set.py +272 -0
  51. {arelle_release-2.37.17.dist-info → arelle_release-2.37.19.dist-info}/WHEEL +0 -0
  52. {arelle_release-2.37.17.dist-info → arelle_release-2.37.19.dist-info}/entry_points.txt +0 -0
  53. {arelle_release-2.37.17.dist-info → arelle_release-2.37.19.dist-info}/licenses/LICENSE.md +0 -0
  54. {arelle_release-2.37.17.dist-info → arelle_release-2.37.19.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,1038 @@
1
+ """
2
+ See COPYRIGHT.md for copyright information.
3
+
4
+ ## Overview
5
+
6
+ The OIM Taxonomy plugin is designed to load taxonomy objects from JSON that adheres to the Open Information
7
+ Model (OIM) Taxonomy Specification.
8
+
9
+ ## Usage Instructions
10
+
11
+ Any import or direct opening of a JSON-specified taxonomy behaves the same as if loading from an xsd taxonomy or xml linkbases
12
+
13
+ For debugging, saves the xsd objects loaded from the OIM taxonomy if
14
+ command line: specify --saveOIMschemafile
15
+ GUIL provide a formula parameter named saveOIMschemafile (value not null or false)x
16
+
17
+ """
18
+
19
+ from typing import TYPE_CHECKING, cast, GenericAlias, Union, _UnionGenericAlias
20
+ from types import UnionType
21
+
22
+ import os, io, json, sys, time, traceback
23
+ import jsonschema
24
+ import regex as re
25
+ from collections import OrderedDict, defaultdict
26
+ from decimal import Decimal
27
+ from arelle.ModelDocument import load, Type, create as createModelDocument
28
+ from arelle.ModelValue import qname, QName
29
+ from arelle.PythonUtil import SEQUENCE_TYPES, OrderedSet
30
+ from arelle.Version import authorLabel, copyrightLabel
31
+ from arelle.XmlUtil import setXmlns
32
+ from arelle import ModelDocument, UrlUtil, XmlValidate
33
+
34
+ # XbrlObject modules contain nested XbrlOBjects and their type objects
35
+
36
+ from .XbrlAbstract import XbrlAbstract
37
+ from .XbrlConcept import XbrlConcept, XbrlDataType, XbrlUnitType
38
+ from .XbrlCube import (XbrlCube, XbrlCubeDimension, XbrlPeriodConstraint, XbrlDateResolution,
39
+ XbrlCubeType, XbrlAllowedCubeDimension, XbrlRequiredCubeRelationship)
40
+ from .XbrlEntity import XbrlEntity
41
+ from .XbrlGroup import XbrlGroup, XbrlGroupContent
42
+ from .XbrlImportedTaxonomy import XbrlImportedTaxonomy
43
+ from .XbrlLabel import XbrlLabel, XbrlLabelType
44
+ from .XbrlNetwork import XbrlNetwork, XbrlRelationship, XbrlRelationshipType
45
+ from .XbrlProperty import XbrlProperty, XbrlPropertyType
46
+ from .XbrlReference import XbrlReference, XbrlReferenceType
47
+ from .XbrlReport import XbrlFact
48
+ from .XbrlTransform import XbrlTransform
49
+ from .XbrlUnit import XbrlUnit
50
+ from .XbrlTaxonomy import XbrlTaxonomy
51
+ from .XbrlTaxonomyObject import XbrlObject, XbrlReferencableTaxonomyObject, XbrlTaxonomyTagObject
52
+ from .XbrlDts import XbrlDts, castToDts
53
+ from .XbrlTypes import XbrlTaxonomyType, QNameKeyType, SQNameKeyType, DefaultTrue, DefaultFalse, DefaultZero
54
+ from .ValidateDTS import validateDTS
55
+ from .ModelValueMore import SQName, QNameAt
56
+ from .ViewXbrlTxmyObj import viewXbrlTxmyObj
57
+ from .XbrlConst import xbrl, oimTaxonomyDocTypePattern, oimTaxonomyDocTypes, qnXbrlLabelObj, xbrlTaxonomyObjects
58
+
59
+
60
+ from arelle.oim.Load import (DUPJSONKEY, DUPJSONVALUE, EMPTY_DICT, EMPTY_LIST, UrlInvalidPattern,
61
+ OIMException, NotOIMException)
62
+
63
+ RESOURCES_DIR = os.path.join(os.path.dirname(__file__), "resources")
64
+ OIMT_SCHEMA = os.path.join(RESOURCES_DIR, "oim-taxonomy-schema.json")
65
+
66
+ saveOIMTaxonomySchemaFiles = False
67
+ SAVE_OIM_SCHEMA_CMDLINE_PARAMETER = "--saveOIMschemafile"
68
+ SAVE_OIM_SCHEMA_FORULA_PARAMETER = qname("saveOIMschemafile", noPrefixIsNoNamespace=True)
69
+ jsonschemaValidator = None
70
+
71
+ EMPTY_SET = set()
72
+
73
+ def jsonGet(tbl, key, default=None):
74
+ if isinstance(tbl, dict):
75
+ return tbl.get(key, default)
76
+ return default
77
+
78
+ def loadOIMTaxonomy(cntlr, error, warning, modelXbrl, oimFile, mappedUri, **kwargs):
79
+ global jsonschemaValidator
80
+ from arelle import ModelDocument, ModelXbrl, XmlUtil
81
+ from arelle.ModelDocument import ModelDocumentReference
82
+ from arelle.ModelValue import qname
83
+
84
+ _return = None # modelDocument or an exception
85
+
86
+ try:
87
+ currentAction = "initializing"
88
+ startingErrorCount = len(modelXbrl.errors) if modelXbrl else 0
89
+ startedAt = time.time()
90
+ documentType = None # set by loadDict
91
+ includeObjects = kwargs.get("includeObjects")
92
+ includeObjectTypes = kwargs.get("includeObjectTypes")
93
+ excludeLabels = kwargs.get("excludeLabels", False)
94
+ followImport = kwargs.get("followImport", True)
95
+
96
+ currentAction = "loading and parsing OIM Taxonomy file"
97
+ loadDictErrors = []
98
+ def ldError(msgCode, msgText, **kwargs):
99
+ loadDictErrors.append((msgCode, msgText, kwargs))
100
+
101
+ def loadDict(keyValuePairs):
102
+ _dict = {}
103
+ _valueKeyDict = {}
104
+ for key, value in keyValuePairs:
105
+ if isinstance(value, dict):
106
+ if key == "documentInfo" and "documentType" in value:
107
+ global documentType
108
+ documentType = value["documentType"]
109
+ if key in ("namespaces", "taxonomy"):
110
+ normalizedDict = {}
111
+ normalizedValueKeyDict = {}
112
+ if DUPJSONKEY in value:
113
+ normalizedDict[DUPJSONKEY] = value[DUPJSONKEY]
114
+ if DUPJSONVALUE in value:
115
+ normalizedDict[DUPJSONVALUE] = value[DUPJSONVALUE]
116
+ for _key, _value in value.items():
117
+ # _key = _key.strip() # per !178 keys have only normalized values, don't normalize key
118
+ # _value = _value.strip()
119
+ if _key in normalizedDict: # don't put the duplicate in the dictionary but report it as error
120
+ if DUPJSONKEY not in normalizedDict:
121
+ normalizedDict[DUPJSONKEY] = []
122
+ normalizedDict[DUPJSONKEY].append((_key, _value, normalizedDict[_key]))
123
+ elif isinstance(_value, SEQUENCE_TYPES):
124
+ normalizedDict[_key] = _value
125
+ else: # do put into dictionary, only report if it's a map object
126
+ normalizedDict[_key] = _value
127
+ if key == "namespaces":
128
+ if _value in normalizedValueKeyDict:
129
+ if DUPJSONVALUE not in normalizedDict:
130
+ normalizedDict[DUPJSONVALUE] = []
131
+ normalizedDict[DUPJSONVALUE].append((_value, _key, normalizedValueKeyDict[_value]))
132
+ else:
133
+ normalizedValueKeyDict[_value] = _key
134
+ if key == "namespaces":
135
+ if not XmlValidate.NCNamePattern.match(_key):
136
+ ldError("{}:invalidJSONStructure",
137
+ _("The %(map)s alias \"%(alias)s\" must be a canonical NCName value"),
138
+ modelObject=modelXbrl, map=key, alias=_key)
139
+ if UrlInvalidPattern.match(_value):
140
+ ldError("{}:invalidJSONStructure",
141
+ _("The %(map)s alias \"%(alias)s\" URI must be a canonical URI value: \"%(URI)s\"."),
142
+ modelObject=modelXbrl, map=key, alias=_key, URI=_value)
143
+ elif not (_value and UrlUtil.isAbsolute(_value)) or UrlInvalidPattern.match(_value):
144
+ ldError("oimce:invalidURI",
145
+ _("The %(map)s \"%(alias)s\" URI is invalid: \"%(URI)s\"."),
146
+ modelObject=modelXbrl, map=key, alias=_key, URI=_value)
147
+ value.clear() # replace with normalized values
148
+ for _key, _value in normalizedDict.items():
149
+ value[_key] = _value
150
+ if DUPJSONKEY in value:
151
+ for _errKey, _errValue, _otherValue in value[DUPJSONKEY]:
152
+ if key in ("namespaces", ):
153
+ ldError("{}:invalidJSON", # {} expanded when loadDictErrors are processed
154
+ _("The %(map)s alias \"%(prefix)s\" is used on uri \"%(uri1)s\" and uri \"\"%(uri2)s."),
155
+ modelObject=modelXbrl, map=key, prefix=_errKey, uri1=_errValue, uri2=_otherValue)
156
+ else:
157
+ ldError("{}:invalidJSON", # {} expanded when loadDictErrors are processed
158
+ _("The %(obj)s key \"%(key)s\" is used on multiple objects."),
159
+ modelObject=modelXbrl, obj=key, key=_errKey)
160
+ del value[DUPJSONKEY]
161
+ if DUPJSONVALUE in value:
162
+ if key in ("namespaces", ):
163
+ for _errValue, _errKey, _otherKey in value[DUPJSONVALUE]:
164
+ ldError("oimce:multipleAliasesForURI",
165
+ _("The \"%(map)s\" value \"%(uri)s\" is used on alias \"%(alias1)s\" and alias \"%(alias2)s\"."),
166
+ modelObject=modelXbrl, map=key, uri=_errValue, alias1=_errKey, alias2=_otherKey)
167
+ del value[DUPJSONVALUE]
168
+ if key in _dict: # don't put the duplicate in the dictionary but report it as error
169
+ if DUPJSONKEY not in _dict:
170
+ _dict[DUPJSONKEY] = []
171
+ _dict[DUPJSONKEY].append((key, value, _dict[key]))
172
+ else: # do put into dictionary, only report if it's a map object
173
+ _dict[key] = value
174
+ '''
175
+ if isinstance(value, str):
176
+ if value in _valueKeyDict:
177
+ if DUPJSONVALUE not in _dict:
178
+ _dict[DUPJSONVALUE] = []
179
+ _dict[DUPJSONVALUE].append((value, key, _valueKeyDict[value]))
180
+ else:
181
+ _valueKeyDict[value] = key
182
+ '''
183
+ return _dict
184
+
185
+ errPrefix = "xbrlte"
186
+ try:
187
+ if isinstance(oimFile, dict) and oimFile.get("documentInfo",{}).get("documentType") in oimTaxonomyDocTypes:
188
+ oimObject = oimFile
189
+ href = "BakedInConstants"
190
+ txFileName = href
191
+ txBase = None
192
+ else:
193
+ _file = modelXbrl.fileSource.file(oimFile, encoding="utf-8-sig")[0]
194
+ href = os.path.basename(oimFile)
195
+ txFileName = oimFile
196
+ txBase = os.path.dirname(oimFile) # import referenced taxonomies
197
+ with _file as f:
198
+ oimObject = json.load(f, object_pairs_hook=loadDict, parse_float=Decimal)
199
+ except UnicodeDecodeError as ex:
200
+ raise OIMException("{}:invalidJSON".format(errPrefix),
201
+ _("File MUST use utf-8 encoding: %(file)s, error %(error)s"),
202
+ sourceFileLine=oimFile, error=str(ex))
203
+ except json.JSONDecodeError as ex:
204
+ raise OIMException("{}:invalidJSON".format(errPrefix),
205
+ "JSON error while %(action)s, %(file)s, error %(error)s",
206
+ file=oimFile, action=currentAction, error=ex)
207
+ # schema validation
208
+ if jsonschemaValidator is None:
209
+ with io.open(OIMT_SCHEMA, mode="rt") as fh:
210
+ jsonschemaValidator = jsonschema.Draft7Validator(json.load(fh))
211
+ try:
212
+ for err in jsonschemaValidator.iter_errors(oimObject) or ():
213
+ path = []
214
+ for p in err.absolute_path:
215
+ path.append(f"[{p}]" if isinstance(p,int) else f"/{p}")
216
+ msg = err.message
217
+ error("jsonschema:oimTaxonomyError",
218
+ _("Error: %(error)s, jsonObj: %(path)s"),
219
+ sourceFileLine=href, error=msg, path="".join(path))
220
+ except (jsonschema.exceptions.SchemaError, jsonschema.exceptions._RefResolutionError, jsonschema.exceptions.UndefinedTypeCheck) as ex:
221
+ msg = str(ex)
222
+ if "PointerToNowhere" in msg:
223
+ msg = msg[:121]
224
+ error("jsonschema:schemaError",
225
+ _("Error in json schema processing: %(error)s"),
226
+ sourceFileLine=href, error=msg)
227
+ documentInfo = jsonGet(oimObject, "documentInfo", {})
228
+ documentType = jsonGet(documentInfo, "documentType")
229
+ taxonomyObj = jsonGet(oimObject, "taxonomy", {})
230
+ isReport = "facts" in oimObject # for test purposes report facts can be in json object
231
+ if not documentType:
232
+ error("oimce:unsupportedDocumentType",
233
+ _("/documentInfo/docType is missing."),
234
+ file=oimFile)
235
+ elif documentType not in oimTaxonomyDocTypes:
236
+ error("oimce:unsupportedDocumentType",
237
+ _("Unrecognized /documentInfo/docType: %(documentType)s"),
238
+ file=oimFile, documentType=documentType)
239
+ return {}
240
+
241
+ # report loadDict errors
242
+ for msgCode, msgText, kwargs in loadDictErrors:
243
+ error(msgCode.format(errPrefix), msgText, sourceFileLine=href, **kwargs)
244
+ del loadDictErrors[:]
245
+
246
+ extensionProperties = {} # key is property QName, value is property path
247
+
248
+ currentAction = "identifying Metadata objects"
249
+
250
+ # create the taxonomy document
251
+ currentAction = "creating schema"
252
+ prevErrLen = len(modelXbrl.errors) # track any xbrl validation errors
253
+ if modelXbrl: # pull loader implementation
254
+ modelXbrl.blockDpmDBrecursion = True
255
+ schemaDoc = _return = createModelDocument(
256
+ modelXbrl,
257
+ Type.SCHEMA,
258
+ txFileName,
259
+ initialComment="loaded from OIM Taxonomy {}".format(mappedUri),
260
+ documentEncoding="utf-8",
261
+ # base=txBase or modelXbrl.entryLoadingUrl
262
+ )
263
+ schemaDoc.inDTS = True
264
+ xbrlDts = castToDts(modelXbrl, isReport)
265
+ else: # API implementation
266
+ xbrlDts = ModelDts.create(
267
+ cntlr.modelManager,
268
+ Type.SCHEMA,
269
+ initialComment="loaded from OIM Taxonomy {}".format(mappedUri),
270
+ base=txBase)
271
+ _return = xbrlDts.modelDocument
272
+ if len(modelXbrl.errors) > prevErrLen:
273
+ error("oime:invalidTaxonomy",
274
+ _("Unable to obtain a valid taxonomy from URLs provided"),
275
+ sourceFileLine=href)
276
+ # first OIM Taxonomy load Baked In objects
277
+ if not xbrlDts.namedObjects and not "loadingBakedInObjects" in kwargs:
278
+ loadOIMTaxonomy(cntlr, error, warning, modelXbrl, xbrlTaxonomyObjects, "BakedInCoreObjects", loadingBakedInObjects=True, **kwargs)
279
+ loadOIMTaxonomy(cntlr, error, warning, modelXbrl, os.path.join(RESOURCES_DIR, "xbrlSpec.json"), "BakedInXbrlSpecObjects", loadingBakedInObjects=True, **kwargs)
280
+ loadOIMTaxonomy(cntlr, error, warning, modelXbrl, os.path.join(RESOURCES_DIR, "types.json"), "BakedInXbrlSpecObjects", loadingBakedInObjects=True, **kwargs)
281
+ loadOIMTaxonomy(cntlr, error, warning, modelXbrl, os.path.join(RESOURCES_DIR, "utr.json"), "BakedInXbrlSpecObjects", loadingBakedInObjects=True, **kwargs)
282
+ loadOIMTaxonomy(cntlr, error, warning, modelXbrl, os.path.join(RESOURCES_DIR, "ref.json"), "BakedInXbrlSpecObjects", loadingBakedInObjects=True, **kwargs)
283
+ loadOIMTaxonomy(cntlr, error, warning, modelXbrl, os.path.join(RESOURCES_DIR, "iso4217.json"), "BakedInXbrlSpecObjects", loadingBakedInObjects=True, **kwargs)
284
+ namespacePrefixes = {}
285
+ prefixNamespaces = {}
286
+ namespaceUrls = {}
287
+ for prefix, ns in documentInfo.get("namespaces", EMPTY_DICT).items():
288
+ if ns and prefix:
289
+ namespacePrefixes[ns] = prefix
290
+ prefixNamespaces[prefix] = ns
291
+ setXmlns(schemaDoc, prefix, ns)
292
+ if "documentNamespace" in documentInfo:
293
+ schemaDoc.targetNamespace = prefixNamespaces.get(documentInfo["documentNamespace"])
294
+ if "urlMapping" in documentInfo:
295
+ for prefix, url in documentInfo["urlMapping"].items():
296
+ namespaceUrls[prefixNamespaces.get(prefix)] = url
297
+ taxonomyName = qname(taxonomyObj.get("name"), prefixNamespaces)
298
+ if not taxonomyName:
299
+ xbrlDts.error("oime:missingQNameProperty",
300
+ _("Taxonomy must have a name (QName) property"),
301
+ sourceFileLine=href)
302
+
303
+ # check extension properties (where metadata specifies CheckPrefix)
304
+ for extPropSQName, extPropertyPath in extensionProperties.items():
305
+ extPropPrefix = extPropSQName.partition(":")[0]
306
+ if extPropPrefix not in prefixNamespaces:
307
+ error("oimte:unboundPrefix",
308
+ _("The extension property QName prefix was not defined in namespaces: %(extensionProperty)s."),
309
+ sourceFileLine=href, extensionProperty=extPropertyPath)
310
+
311
+ for iImpTxmy, impTxmyObj in enumerate(taxonomyObj.get("importedTaxonomies", EMPTY_LIST)):
312
+ if qnXbrlLabelObj in getattr(impTxmyObj, "includeObjectTypes",()):
313
+ impTxmyObj.includeObjectTypes.delete(qnXbrlLabelObj)
314
+ xbrlDts.error("oimte:invalidObjectType",
315
+ _("/taxonomy/importedTaxonomies[%(iImpTxmy)s] must not have a label object in the includeObjectTypes property"),
316
+ sourceFileLine=href, index=iImpTxmy)
317
+ impTxmyName = qname(impTxmyObj.get("taxonomyName"), prefixNamespaces)
318
+ if impTxmyName:
319
+ ns = impTxmyName.namespaceURI
320
+ # if already imported ignore it (for now)
321
+ if ns not in xbrlDts.namespaceDocs and followImport:
322
+ url = namespaceUrls.get(ns)
323
+ if url:
324
+ load(xbrlDts, url, base=oimFile, isDiscovered=schemaDoc.inDTS, isIncluded=kwargs.get("isIncluded"), namespace=ns,
325
+ includeObjects=(qname(qn, prefixNamespaces) for qn in impTxmyObj.get("includeObjects",())) or None,
326
+ includeObjectTypes=(qname(qn, prefixNamespaces) for qn in impTxmyObj.get("includeObjectTypes",())) or None,
327
+ excludeLabels=impTxmyObj.get("excludeLabels",False),
328
+ followImport=impTxmyObj.get("followImport",True))
329
+ else:
330
+ xbrlDts.error("oime:missingQNameProperty",
331
+ _("/taxonomy/importedTaxonomies[%(iImpTxmy)s] must have a taxonomyName (QName) property"),
332
+ sourceFileLine=href, index=iImpTxmy)
333
+ def singular(name):
334
+ if name.endswith("ies"):
335
+ return name[:-3] + "y"
336
+ elif name.endswith("s"):
337
+ return name[:-1]
338
+ return name
339
+ def plural(name):
340
+ if name.endswith("y"):
341
+ return name[:-1] + "ies"
342
+ else:
343
+ return name + "s"
344
+ def addToCol(oimParentObj, objName, newObj, key):
345
+ parentCol = getattr(oimParentObj, plural(objName), None) # parent collection object
346
+ if colObj is not None:
347
+ if key:
348
+ colObj[key] = newObj
349
+ else:
350
+ colObj.add(newObj)
351
+
352
+ jsonEltsNotInObjClass = []
353
+ jsonEltsReqdButMissing = []
354
+ namedObjectDuplicates = defaultdict(OrderedSet)
355
+ def createTaxonomyObject(jsonObj, oimParentObj, keyClass, objClass, newObj, pathParts):
356
+ keyValue = None
357
+ relatedNames = [] # to tag an object with labels or references
358
+ unexpectedJsonProps = set(jsonObj.keys())
359
+ for propName, propType in getattr(objClass, "__annotations__", EMPTY_DICT).items():
360
+ if isinstance(propType, GenericAlias):
361
+ propClass = propType.__origin__ # collection type such as OrderedSet, dict
362
+ collectionProp = propClass()
363
+ setattr(newObj, propName, collectionProp) # fresh new dict or OrderedSet (even if no contents for it)
364
+ else:
365
+ propClass = propType
366
+ if propName in jsonObj:
367
+ unexpectedJsonProps.remove(propName)
368
+ if propName == "labels" and excludeLabels:
369
+ continue
370
+ jsonValue = jsonObj[propName]
371
+ if isinstance(propType, GenericAlias):
372
+ if len(propType.__args__) == 2: # dict
373
+ _keyClass = propType.__args__[0] # class of key such as QNameKey
374
+ eltClass = propType.__args__[1] # class of collection elements such as XbrlConcept
375
+ elif len(propType.__args__) == 1: # set such as OrderedSet or list
376
+ _keyClass = None
377
+ eltClass = propType.__args__[0]
378
+ if isinstance(jsonValue, list):
379
+ for iObj, listObj in enumerate(jsonValue):
380
+ if isinstance(eltClass, str) or eltClass.__name__.startswith("Xbrl"): # nested Xbrl objects
381
+ if isinstance(listObj, dict):
382
+ # this handles lists of dict objects. For dicts of key-value dict objects see above.
383
+ createTaxonomyObjects(propName, listObj, newObj, pathParts + [f'{propName}[{iObj}]'])
384
+ else:
385
+ error("xbrlte:invalidObjectType",
386
+ _("Object expected but non-object found: %(listObj)s, jsonObj: %(path)s"),
387
+ sourceFileLine=href, listObj=listObj, path=f"{'/'.join(pathParts + [f'{propName}[{iObj}]'])}")
388
+ else: # collection contains ordinary values
389
+ if eltClass in (QName, QNameKeyType, SQName, SQNameKeyType):
390
+ listObj = qname(listObj, prefixNamespaces)
391
+ if listObj is None:
392
+ error("xbrlte:invalidQName",
393
+ _("QName is invalid: %(qname)s, jsonObj: %(path)s"),
394
+ sourceFileLine=href, qname=jsonObj[propName], path=f"{'/'.join(pathParts + [f'{propName}[{iObj}]'])}")
395
+ continue # skip this property
396
+ if propName == "relatedNames":
397
+ relatedNames.append(listObj)
398
+ if propClass in (set, OrderedSet):
399
+ collectionProp.add(listObj)
400
+ else:
401
+ collectionProp.append(listObj)
402
+ elif isinstance(jsonValue, dict) and keyClass:
403
+ for iObj, (valKey, valVal) in enumerate(jsonValue.items()):
404
+ if isinstance(_keyClass, UnionType):
405
+ if QName in _keyClass.__args__ and ":" in valKey:
406
+ _valKey = qname(listObj, prefixNamespaces)
407
+ if _valKey is None:
408
+ error("xbrlte:invalidQName",
409
+ _("QName is invalid: %(qname)s, jsonObj: %(path)s"),
410
+ sourceFileLine=href, qname=_valKey, path=f"{'/'.join(pathParts + [f'{propName}[{iObj}]'])}")
411
+ continue # skip this property
412
+ elif str in _keyClass.__args__:
413
+ _valKey = valKey
414
+ else:
415
+ continue
416
+ elif isinstance(_keyClass, str):
417
+ _valKey = valKey
418
+ else:
419
+ continue
420
+ collectionProp[_valKey] = valVal
421
+ elif isinstance(propType, _UnionGenericAlias) and propType.__args__[-1] == type(None) and isinstance(jsonValue,dict): # optional embdded object
422
+ createTaxonomyObjects(propName, jsonValue, newObj, pathParts + [propName]) # object property
423
+ else:
424
+ optional = False
425
+ if isinstance(propType, _UnionGenericAlias) and propType.__args__[-1] == type(None):
426
+ propType = propType.__args__[0] # scalar property
427
+ optional = True
428
+ if propType == QNameAt:
429
+ jsonValue, _sep, atSuffix = jsonValue.partition("@")
430
+ if propType in (QName, QNameKeyType, SQName, SQNameKeyType, QNameAt):
431
+ jsonValue = qname(jsonValue, prefixNamespaces)
432
+ if jsonValue is None:
433
+ error("xbrlte:invalidQName",
434
+ _("QName is invalid: %(qname)s, jsonObj: %(path)s"),
435
+ sourceFileLine=href, qname=jsonObj[propName], path=f"{'/'.join(pathParts + [propName])}")
436
+ if optional:
437
+ jsonValue = None
438
+ else:
439
+ # TBD: set newObj to invalid so it isn't usable (it was already set in collection objects above when being created)
440
+ return # skip this nested object entirely
441
+ elif propType == QNameAt:
442
+ jsonValue = QNameAt(jsonValue.prefix, jsonValue.namespaceURI, jsonValue.localName, atSuffix)
443
+ if propName == "relatedName":
444
+ relatedNames.append(jsonValue)
445
+ setattr(newObj, propName, jsonValue)
446
+ if (keyClass and keyClass == propType) or (not keyClass and propType in (QNameKeyType, SQNameKeyType)):
447
+ keyValue = jsonValue # e.g. the QNAme of the new object for parent object collection
448
+ elif propType in (type(oimParentObj), type(oimParentObj).__name__): # propType may be a TypeAlias which is a string name of class
449
+ setattr(newObj, propName, oimParentObj)
450
+ elif ((isinstance(propType, (UnionType,_UnionGenericAlias)) or isinstance(getattr(propType, "__origin__", None), type(Union))) and # Optional[ ] type
451
+ propType.__args__[-1] in (type(None), DefaultTrue, DefaultFalse, DefaultZero)):
452
+ setattr(newObj, propName, {type(None): None, DefaultTrue: True, DefaultFalse: False, DefaultZero:0}[propType.__args__[-1]]) # use first of union for prop value creation
453
+ else: # absent json element
454
+ if not propClass in (dict, set, OrderedSet, OrderedDict):
455
+ jsonEltsReqdButMissing.append(f"{'/'.join(pathParts + [propName])}")
456
+ setattr(newObj, propName, None) # not defaultable but set to None anyway
457
+ if unexpectedJsonProps:
458
+ for propName in unexpectedJsonProps:
459
+ jsonEltsNotInObjClass.append(f"{'/'.join(pathParts + [propName])}={jsonObj.get(propName,'(absent)')}")
460
+ if isinstance(newObj, XbrlReferencableTaxonomyObject):
461
+ if keyValue in xbrlDts.namedObjects:
462
+ namedObjectDuplicates[keyValue].add(newObj)
463
+ namedObjectDuplicates[keyValue].add(xbrlDts.namedObjects[keyValue])
464
+ else:
465
+ xbrlDts.namedObjects[keyValue] = newObj
466
+ elif isinstance(newObj, XbrlTaxonomyTagObject) and relatedNames:
467
+ for relatedQn in relatedNames:
468
+ xbrlDts.tagObjects[relatedQn].append(newObj)
469
+ return keyValue
470
+
471
+ def createTaxonomyObjects(jsonKey, jsonObj, oimParentObj, pathParts):
472
+ # find collection owner in oimParentObj
473
+ for objName in (jsonKey, plural(jsonKey)):
474
+ ownrPropType = getattr(oimParentObj, "__annotations__", EMPTY_DICT).get(objName)
475
+ if ownrPropType is not None:
476
+ break
477
+ if ownrPropType is not None:
478
+ ownrProp = getattr(oimParentObj, objName, None) # owner collection or property
479
+ if ownrPropType is not None:
480
+ if isinstance(ownrPropType, GenericAlias):
481
+ ownrPropClass = ownrPropType.__origin__ # collection type such as OrderedSet, dict
482
+ if len(ownrPropType.__args__) == 2: # dict
483
+ keyClass = ownrPropType.__args__[0] # class of key such as QNameKey
484
+ objClass = ownrPropType.__args__[1] # class of obj such as XbrlConcept
485
+ elif len(ownrPropType.__args__) == 1: # set such as OrderedSet or list
486
+ keyClass = None
487
+ objClass = ownrPropType.__args__[0]
488
+ if ownrProp is None: # the parent object's dict or OrderedSet doesn't exist yet
489
+ ownrProp = ownrPropClass()
490
+ setattr(oimParentObj, propName, ownrProp) # fresh new dict or OrderedSet
491
+ if objClass == XbrlFact and isinstance(jsonObj, dict): # this is a JSON key-value dict going into a dict, for lists of dicts see below
492
+ for id, value in jsonObj.items():
493
+ newObj = objClass(dtsObjectIndex=len(xbrlDts.xbrlObjects))
494
+ xbrlDts.xbrlObjects.append(newObj)
495
+ newObj.id = id
496
+ createTaxonomyObject(value, oimParentObj, str, objClass, newObj, pathParts + [f"[{id}]"])
497
+ ownrProp[id] = newObj
498
+ return None # facts not returnable
499
+ elif isinstance(ownrPropType, _UnionGenericAlias) and ownrPropType.__args__[-1] == type(None): # optional nested object
500
+ keyClass = None
501
+ objClass = ownrPropType.__args__[0]
502
+ else: # parent is just an object field, not a collection
503
+ objClass = ownrPropType # e.g just a Concept but no owning collection
504
+ if objClass == XbrlTaxonomyType:
505
+ objClass = XbrlTaxonomy
506
+ if issubclass(objClass, XbrlObject):
507
+ newObj = objClass(dtsObjectIndex=len(xbrlDts.xbrlObjects)) # e.g. this is the new Concept
508
+ xbrlDts.xbrlObjects.append(newObj)
509
+ classCountProp = f"_{objClass.__name__}Count"
510
+ classIndex = getattr(oimParentObj, classCountProp, 0)
511
+ setattr(newObj, "_classIndex", classIndex)
512
+ setattr(oimParentObj, classCountProp,classIndex+1)
513
+ else:
514
+ newObj = objClass() # e.g. XbrlProperty
515
+ keyValue = createTaxonomyObject(jsonObj, oimParentObj, keyClass, objClass, newObj, pathParts)
516
+ if isinstance(ownrPropType, GenericAlias):
517
+ if len(ownrPropType.__args__) == 2:
518
+ if keyValue:
519
+ ownrProp[keyValue] = newObj
520
+ elif isinstance(ownrProp, (set, OrderedSet)):
521
+ ownrProp.add(newObj)
522
+ else:
523
+ ownrProp.append(newObj)
524
+ elif isinstance(ownrPropType, _UnionGenericAlias) and ownrPropType.__args__[-1] == type(None): # optional nested object
525
+ setattr(oimParentObj, pathParts[-1], newObj)
526
+ return newObj
527
+ return None
528
+
529
+ newTxmy = createTaxonomyObjects("taxonomy", oimObject["taxonomy"], xbrlDts, ["", "taxonomy"])
530
+ if isReport:
531
+ createTaxonomyObjects("facts", oimObject["facts"], xbrlDts, ["", "facts"])
532
+
533
+ if jsonEltsNotInObjClass:
534
+ error("arelle:undeclaredOimTaxonomyJsonElements",
535
+ _("Json file has elements not declared in Arelle object classes: %(undeclaredElements)s"),
536
+ sourceFileLine=href, undeclaredElements=", ".join(jsonEltsNotInObjClass))
537
+ if jsonEltsReqdButMissing:
538
+ error("arelle:missingOimTaxonomyJsonElements",
539
+ _("Json file missing required elements: %(missingElements)s"),
540
+ sourceFileLine=href, missingElements=", ".join(jsonEltsReqdButMissing))
541
+
542
+ for qname, dupObjs in namedObjectDuplicates.items():
543
+ xbrlDts.error("oimte:duplicateObjects",
544
+ _("Multiple referenceable objects have the same name: %(qname)s"),
545
+ xbrlObject=dupObjs, qname=qname)
546
+
547
+ if newTxmy is not None:
548
+ for impTxmy in newTxmy.importedTaxonomies:
549
+ if impTxmy.taxonomyName not in xbrlDts.taxonomies:
550
+ # is it present in urlMapping?
551
+ url = namespaceUrls.get(impTxmy.taxonomyName.prefix)
552
+ if url:
553
+ loadOIMTaxonomy(cntlr, error, warning, modelXbrl, url, impTxmy.taxonomyName.localName)
554
+
555
+ xbrlDts.namespaceDocs[taxonomyName.namespaceURI].append(schemaDoc)
556
+
557
+ return schemaDoc
558
+
559
+ ####################### convert to XML Taxonomy
560
+
561
+
562
+ QN_ANNOTATION = qname("{http://www.w3.org/2001/XMLSchema}xs:annotation")
563
+ QN_APPINFO = qname("{http://www.w3.org/2001/XMLSchema}xs:appinfo")
564
+ QN_IMPORT = qname("{http://www.w3.org/2001/XMLSchema}xs:import")
565
+ QN_ELEMENT = qname("{http://www.w3.org/2001/XMLSchema}xs:element")
566
+ QN_PERIOD_TYPE = qname("{http://www.xbrl.org/2003/instance}xbrli:periodType")
567
+ QN_BALANCE = qname("{http://www.xbrl.org/2003/instance}xbrli:balance")
568
+ QN_SUBS_GROUP = qname("{http://www.w3.org/2001/XMLSchema}xs:substitutionGroup")
569
+ QN_ROLE_TYPE = qname("{http://www.xbrl.org/2003/linkbase}link:roleType")
570
+ QN_ROLE_TYPE = qname("{http://www.xbrl.org/2003/linkbase}link:roleType")
571
+ QN_DEFINITION = qname("{http://www.xbrl.org/2003/linkbase}link:definition")
572
+ QN_USED_ON = qname("{http://www.xbrl.org/2003/linkbase}link:usedOn")
573
+
574
+ # convert into XML Taxonomy
575
+ schemaElt = schemaDoc.xmlRootElement
576
+ annotationElt = addChild(schemaElt, QN_ANNOTATION)
577
+ appinfoElt = addChild(annotationElt, QN_APPINFO)
578
+
579
+ if not any(t["namespace"] == "http://www.xbrl.org/2003/instance" for t in taxonomyRefs):
580
+ taxonomyRefs.insert(0, {"namespace":"http://www.xbrl.org/2003/instance", "entryPoint":"http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd"})
581
+ if "cubes" in taxonomyObj and not any(t["namespace"] == "http://xbrl.org/2005/xbrldt" for t in taxonomyRefs):
582
+ taxonomyRefs.insert(0, {"namespace":"http://xbrl.org/2005/xbrldt", "entryPoint": "http://www.xbrl.org/2005/xbrldt-2005.xsd"})
583
+ for txmyRefObj in taxonomyRefs:
584
+ schemaDoc.addDocumentReference(
585
+ loadModelDocument(modelXbrl, txmyRefObj.get("entryPoint"), txBase, namespace=txmyRefObj.get("namespace"), isDiscovered=True),
586
+ "import")
587
+ addChild(schemaElt, QN_IMPORT, attributes={
588
+ "namespace": txmyRefObj.get("namespace"),
589
+ "schemaLocation": txmyRefObj.get("entryPoint")})
590
+
591
+ # additional namespaces needed
592
+ for prefix, ns in (("xlink", "http://www.w3.org/1999/xlink"),
593
+ ("ref", "http://www.xbrl.org/2006/ref"),
594
+ ("xbrldt", "http://xbrl.org/2005/xbrldt"),
595
+ ("xbrl", xbrl)):
596
+ if ns not in namespacePrefixes:
597
+ namespacePrefixes[ns] = prefix
598
+ prefixNamespaces[prefix] = ns
599
+ setXmlns(schemaDoc, prefix, ns)
600
+
601
+
602
+
603
+ ##### Move to end
604
+ currentAction = "identifying default dimensions"
605
+ if modelXbrl is not None:
606
+ ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # needs dimension defaults
607
+
608
+ currentAction = "creating concepts"
609
+ conceptObjs = taxonomyObj.get("concepts", [])
610
+ conceptNum = 0 # for synthetic concept number
611
+ syntheticConceptFormat = "_f{{:0{}}}".format(int(log10(len(conceptObjs) or 1))) #want
612
+
613
+ numConceptCreationXbrlErrors = 0
614
+
615
+ for conceptI, conceptObj in enumerate(conceptObjs):
616
+ nillable = conceptObj.get("nillable", True)
617
+ periodType = conceptObj.get("periodType", True)
618
+ balance = conceptObj.get("balance", None)
619
+ abstract = conceptObj.get("abstract", None)
620
+ nillable = conceptObj.get("abstract", None)
621
+ name = qname(conceptObj.get("name", ""), prefixNamespaces)
622
+ if name:
623
+ dataType = qname(conceptObj.get("dataType", ""), prefixNamespaces)
624
+ substitutionGroup = qname(conceptObj.get("substitutionGroup", ""), prefixNamespaces)
625
+ attributes = {"id": f"{name.prefix}_{name.localName}",
626
+ "name": name.localName}
627
+ if dataType:
628
+ attributes["type"] = str(dataType)
629
+ if periodType:
630
+ attributes[QN_PERIOD_TYPE.clarkNotation] = periodType
631
+ if balance:
632
+ attributes[QN_BALANCE.clarkNotation] = balance
633
+ if abstract is not None:
634
+ attributes["abstract"] = str(abstract).lower()
635
+ if nillable is not None:
636
+ attributes["nillable"] = str(nillable).lower()
637
+ attributes["substitutionGroup"] = str(substitutionGroup)
638
+
639
+ conceptElt = addChild(schemaElt,
640
+ QN_ELEMENT,
641
+ attributes=attributes)
642
+ else:
643
+ error("oimte:invalidConceptName",
644
+ "%(path)s concept name %(name)s is not valid",
645
+ modelObject=modelXbrl, path=f"taxonomy/concept[{conceptI+1}]", name=conceptObj.get("name", ""))
646
+
647
+ # find which linkbases networkURIs are used on
648
+ usedOn = defaultdict(set)
649
+ for dimObj in taxonomyObj.get("cubes", []) + taxonomyObj.get("domains", []):
650
+ networkURI = dimObj.get("networkURI", "")
651
+ if networkURI:
652
+ usedOn[networkURI].add("link:definitionLink")
653
+ for networkObj in taxonomyObj.get("networks", []) + taxonomyObj.get("domains", []):
654
+ networkURI = networkObj.get("networkURI", "")
655
+ if networkURI:
656
+ for relObj in networkObj.get("relationships", []):
657
+ relType = relObj.get("relationshipType", "")
658
+ if relType == XbrlConst.parentChild:
659
+ usedOn[networkURI].add("link:presentationLink")
660
+ elif relType in XbrlConst.summationItems:
661
+ usedOn[networkURI].add("link:calculationLink")
662
+ elif relType in (XbrlConst.requiresElement, XbrlConst.generalSpecial):
663
+ usedOn[networkURI].add("link:definitionLink")
664
+ for networkObj in taxonomyObj.get("cubes", []):
665
+ networkURI = networkObj.get("networkURI", "")
666
+ if networkURI and networkObj.get("dimensions", []):
667
+ usedOn[networkURI].add("link:definitionLink")
668
+
669
+ # define role types
670
+ locallyDefinedRoles = OrderedDict() # URI: {name, description, usedOns}
671
+ # networkURI can appear multiple times in different places, infer a single role definition from all
672
+ for objI, networkObj in enumerate(taxonomyObj.get("networks", []) + taxonomyObj.get("domains", [])):
673
+ networkURI = networkObj.get("networkURI", "")
674
+ if networkURI in locallyDefinedRoles:
675
+ roleDef = locallyDefinedRoles[networkURI]
676
+ else:
677
+ locallyDefinedRoles[networkURI] = roleDef = {}
678
+ if "name" in networkObj:
679
+ roleDef["name"] = networkObj["name"]
680
+ if "description" in networkObj:
681
+ roleDev["description"] = networkObj["description"]
682
+ locallyDefinedRoleHrefs = {}
683
+ for objI, (networkURI, networkObj) in enumerate(locallyDefinedRoles.items()):
684
+ name = networkObj.get("name", f"_roleType_{objI+1}")
685
+ description = networkObj.get("description", "")
686
+ locallyDefinedRoleHrefs[networkURI] = f"#{name}"
687
+
688
+ roleTypeElt = addChild(appinfoElt, QN_ROLE_TYPE,
689
+ attributes={"id": name,
690
+ "roleURI": networkURI})
691
+ if description:
692
+ addChild(roleTypeElt, QN_DEFINITION, text=description)
693
+ for u in usedOn[networkURI]:
694
+ addChild(roleTypeElt, QN_USED_ON, text=u)
695
+ modelXbrl.roleTypes[networkURI].append(roleTypeElt)
696
+
697
+ # create ELRs
698
+ lbElts = []
699
+ xlinkLabelFormat = "{{}}{{:0{}}}".format(int(log10(len(taxonomyObj.get("labels", [])) or 1)))
700
+ locXlinkLabels = {}
701
+ hrefsNsWithoutPrefix = defaultdict(list)
702
+ def locXlinkLabel(elrElt, conceptRef, path):
703
+ if conceptRef not in locXlinkLabels:
704
+ qn = qname(conceptRef, prefixNamespaces)
705
+ if qn is None:
706
+ error("oimte:invalidConceptRef",
707
+ "%(path)s concept reference %(conceptRef)s is not a defined prefix or not a valid qname",
708
+ modelObject=modelXbrl, path=path, conceptRef=conceptRef)
709
+ elif qn.namespaceURI not in namespacePrefixes:
710
+ hrefsNsWithoutPrefix[qn.namespaceURI or "(none)"].append(path)
711
+ concept = modelXbrl.qnameConcepts.get(qn)
712
+ xlinkLabel = xlinkLabelFormat.format("loc", len(locXlinkLabels)+1)
713
+ locXlinkLabels[conceptRef] = xlinkLabel
714
+ if concept is not None:
715
+ addChild(elrElt, XbrlConst.qnLinkLoc,
716
+ attributes={"{http://www.w3.org/1999/xlink}label": xlinkLabel,
717
+ "{http://www.w3.org/1999/xlink}href": f"{concept.modelDocument.uri}#{concept.id}",
718
+ "{http://www.w3.org/1999/xlink}type": "locator"})
719
+ else:
720
+ error("oimte:invalidConceptQName",
721
+ "%(path)s concept reference %(conceptRef)s, qname %(qname)s, does not correspond to a defined concept",
722
+ modelObject=modelXbrl, path=path, qname=qn, conceptRef=conceptRef)
723
+ return locXlinkLabels[conceptRef]
724
+ lbElt = addChild(appinfoElt, XbrlConst.qnLinkLinkbase,
725
+ attributes={"id":"_labels_"})
726
+ lbElts.append(lbElt)
727
+ elrElt = addChild(lbElt, XbrlConst.qnLinkLabelLink,
728
+ attributes={"{http://www.w3.org/1999/xlink}role": XbrlConst.defaultLinkRole,
729
+ "{http://www.w3.org/1999/xlink}type": "extended"})
730
+ for labelI, labelObj in enumerate(taxonomyObj.get("labels", [])):
731
+ labelType = labelObj.get("labelType", "")
732
+ value = labelObj.get("value", "")
733
+ language = labelObj.get("language", "")
734
+ relatedID = labelObj.get("relatedID", [])
735
+ xlinkLbl = xlinkLabelFormat.format("label", labelI+1)
736
+ addChild(elrElt, XbrlConst.qnLinkLabel, text=value,
737
+ attributes={# "id": xlinkLbl,
738
+ "{http://www.w3.org/1999/xlink}label": xlinkLbl,
739
+ "{http://www.w3.org/1999/xlink}role": labelType,
740
+ "{http://www.w3.org/1999/xlink}lang": language,
741
+ "{http://www.w3.org/1999/xlink}type": "resource"})
742
+ for refI, ref in enumerate(relatedID):
743
+ addChild(elrElt, XbrlConst.qnLinkLabelArc,
744
+ attributes={"{http://www.w3.org/1999/xlink}from": locXlinkLabel(elrElt, ref, f"label[{labelI}]/relatedID[{refI}]"),
745
+ "{http://www.w3.org/1999/xlink}to": xlinkLbl,
746
+ "{http://www.w3.org/1999/xlink}arcrole": XbrlConst.conceptLabel,
747
+ "{http://www.w3.org/1999/xlink}type": "arc"})
748
+
749
+ def addRoleRefs(lbElt, roles, arcroles):
750
+ firstElr = lbElt[0]
751
+ for role in sorted(roles):
752
+ if role in locallyDefinedRoleHrefs:
753
+ href = locallyDefinedRoleHrefs[role]
754
+ addChild(lbElt, XbrlConst.qnLinkRoleRef,
755
+ beforeSibling=firstElr,
756
+ attributes={
757
+ "{http://www.w3.org/1999/xlink}roleURI": role,
758
+ "{http://www.w3.org/1999/xlink}type": "simple",
759
+ "{http://www.w3.org/1999/xlink}href": href})
760
+ for arcrole in sorted(arcroles):
761
+ if arcrole.startswith("http://xbrl.org/int/dim/arcrole"):
762
+ href = f"http://www.xbrl.org/2005/xbrldt-2005.xsd#{os.path.basename(arcrole)}"
763
+ addChild(lbElt, XbrlConst.qnLinkArcroleRef,
764
+ beforeSibling=firstElr,
765
+ attributes={
766
+ "{http://www.w3.org/1999/xlink}arcroleURI": arcrole,
767
+ "{http://www.w3.org/1999/xlink}type": "simple",
768
+ "{http://www.w3.org/1999/xlink}href": href})
769
+ roles.clear()
770
+ arcroles.clear()
771
+
772
+ domainIDHypercubeQNames = {}
773
+ domainIDPrimaryDimensions = {}
774
+ domainIDPeriodDimensions = {}
775
+ lbElt = addChild(appinfoElt, XbrlConst.qnLinkLinkbase)
776
+ lbElts.append(lbElt)
777
+ lbEltRoleRefs = set()
778
+ lbEltArcroleRefs = set()
779
+ for cubeI, cubeObj in enumerate(taxonomyObj.get("cubes", [])):
780
+ locXlinkLabels.clear() # separate locs per elr
781
+ networkURI = cubeObj.get("networkURI", "") # ELR
782
+ hypercubeConcept = cubeObj.get("name", "") # hypercube concept clark name
783
+ cubeType = cubeObj.get("cubeType", "")
784
+ elrElt = addChild(lbElt, XbrlConst.qnLinkDefinitionLink,
785
+ attributes={"{http://www.w3.org/1999/xlink}role": networkURI,
786
+ "{http://www.w3.org/1999/xlink}type": "extended"})
787
+ lbEltRoleRefs.add(networkURI)
788
+ for dimI, dimObj in enumerate(cubeObj.get("dimensions", [])):
789
+ dimensionType = dimObj.get("dimensionType", "")
790
+ domainID = dimObj.get("domainID", "")
791
+ dimensionType = dimObj.get("dimensionType", "")
792
+ dimensionConcept = dimObj.get("dimensionConcept", "")
793
+ if dimensionConcept == "xbrl:PrimaryDimension":
794
+ domainIDPrimaryDimensions[domainID] = hypercubeConcept
795
+ elif dimensionConcept == "xbrl:PeriodDimension":
796
+ domainIDPeriodDimensions[domainID] = hypercubeConcept
797
+ else:
798
+ domainIDHypercubeQNames[domainID] = hypercubeConcept
799
+ addChild(elrElt, XbrlConst.qnLinkDefinitionArc,
800
+ attributes={"{http://www.w3.org/1999/xlink}from": locXlinkLabel(elrElt, hypercubeConcept, f"cube[{cubeI}]/cube.name"),
801
+ "{http://www.w3.org/1999/xlink}to": locXlinkLabel(elrElt, dimensionConcept, f"cube[{cubeI}]/dimension[{dimI}]/dimensionConcept"),
802
+ "{http://www.w3.org/1999/xlink}arcrole": XbrlConst.hypercubeDimension,
803
+ "{http://www.w3.org/1999/xlink}type": "arc"})
804
+ lbEltArcroleRefs.add(XbrlConst.hypercubeDimension)
805
+
806
+ for domI, domObj in enumerate(taxonomyObj.get("domains", [])):
807
+ locXlinkLabels.clear() # separate locs per elr
808
+ networkURI = domObj.get("networkURI", "")
809
+ domainID = domObj.get("domainID", "")
810
+ domainConcept = domObj.get("domainConcept", "")
811
+ relationships = domObj.get("relationships", [])
812
+ elrElt = addChild(lbElt, XbrlConst.qnLinkDefinitionLink,
813
+ attributes={"{http://www.w3.org/1999/xlink}role": networkURI,
814
+ "{http://www.w3.org/1999/xlink}type": "extended"})
815
+ lbEltRoleRefs.add(networkURI)
816
+ if domainID not in domainIDPrimaryDimensions and domainID not in domainIDPeriodDimensions:
817
+ addChild(elrElt, XbrlConst.qnLinkDefinitionArc,
818
+ attributes={"{http://www.w3.org/1999/xlink}from": locXlinkLabel(elrElt, domainIDHypercubeQNames.get(domainID), f"domain[{domI}]/domainID"),
819
+ "{http://www.w3.org/1999/xlink}to": locXlinkLabel(elrElt, domainConcept, f"domain[{domI}]/domainConcept"),
820
+ "{http://www.w3.org/1999/xlink}arcrole": XbrlConst.dimensionDomain,
821
+ "{http://www.w3.org/1999/xlink}type": "arc"})
822
+ lbEltArcroleRefs.add(XbrlConst.dimensionDomain)
823
+ for relI, relObj in enumerate(relationships):
824
+ source = relObj.get("source", "")
825
+ target = relObj.get("target", "")
826
+ order = relObj.get("order", "1")
827
+ if domainID in domainIDPrimaryDimensions and relI == 0:
828
+ addChild(elrElt, XbrlConst.qnLinkDefinitionArc,
829
+ attributes={"{http://www.w3.org/1999/xlink}from": locXlinkLabel(elrElt, target, f"domain[{domI}]/relationship[{relI}/target"),
830
+ "{http://www.w3.org/1999/xlink}to": locXlinkLabel(elrElt, domainIDPrimaryDimensions.get(domainID), f"domain[{domI}]/domainID"),
831
+ "{http://www.w3.org/1999/xlink}arcrole": XbrlConst.all,
832
+ "{http://www.w3.org/1999/xlink}type": "arc",
833
+ # TBD - determine values dynamically from taxonomy and authority
834
+ "{http://xbrl.org/2005/xbrldt}closed": "true",
835
+ "{http://xbrl.org/2005/xbrldt}contextElement": "segment"})
836
+ lbEltArcroleRefs.add(XbrlConst.all)
837
+ else:
838
+ addChild(elrElt, XbrlConst.qnLinkDefinitionArc,
839
+ attributes={"{http://www.w3.org/1999/xlink}from": locXlinkLabel(elrElt, source, f"domain[{domI}]/relationship[{relI}/source"),
840
+ "{http://www.w3.org/1999/xlink}to": locXlinkLabel(elrElt, target, f"domain[{domI}]/relationship[{relI}/target"),
841
+ "{http://www.w3.org/1999/xlink}arcrole": XbrlConst.domainMember,
842
+ "{http://www.w3.org/1999/xlink}type": "arc",
843
+ "order": order})
844
+ lbEltArcroleRefs.add(XbrlConst.domainMember)
845
+ addRoleRefs(lbElt, lbEltRoleRefs, lbEltArcroleRefs)
846
+
847
+ lbElt = addChild(appinfoElt, XbrlConst.qnLinkLinkbase)
848
+ lbElts.append(lbElt)
849
+ for networkI, networkObj in enumerate(taxonomyObj.get("networks", [])):
850
+ locXlinkLabels.clear() # separate locs per elr
851
+ networkURI = networkObj.get("networkURI", "")
852
+ elrElt = addChild(lbElt, XbrlConst.qnLinkDefinitionLink,
853
+ attributes={"{http://www.w3.org/1999/xlink}role": networkURI,
854
+ "{http://www.w3.org/1999/xlink}type": "extended"})
855
+ lbEltRoleRefs.add(networkURI)
856
+ relationships = networkObj.get("relationships", [])
857
+ for relI, relObj in enumerate(relationships):
858
+ source = relObj.get("source", "")
859
+ target = relObj.get("target", "")
860
+ order = relObj.get("order", None)
861
+ relationshipType = relObj.get("relationshipType", "")
862
+ preferredLabel = relObj.get("preferredLabel", None)
863
+ weight = relObj.get("weight", None)
864
+ attributes = {"{http://www.w3.org/1999/xlink}from": locXlinkLabel(elrElt, source, f"network[{networkI}]/relationship[{relI}/source"),
865
+ "{http://www.w3.org/1999/xlink}to": locXlinkLabel(elrElt, target, f"network[{networkI}]/relationship[{relI}/target"),
866
+ "{http://www.w3.org/1999/xlink}arcrole": relationshipType,
867
+ "{http://www.w3.org/1999/xlink}type": "arc"}
868
+ lbEltArcroleRefs.add(relationshipType)
869
+ if weight is not None:
870
+ attributes["weight"] = weight
871
+ if preferredLabel is not None:
872
+ attributes["preferredLabel"] = preferredLabel
873
+ addChild(elrElt, XbrlConst.qnLinkDefinitionArc, attributes)
874
+ addRoleRefs(lbElt, lbEltRoleRefs, lbEltArcroleRefs)
875
+
876
+ locXlinkLabels.clear() # separate locs per elr
877
+ elrElt = addChild(lbElt, XbrlConst.qnLinkReferenceLink,
878
+ attributes={"{http://www.w3.org/1999/xlink}role": XbrlConst.defaultLinkRole,
879
+ "{http://www.w3.org/1999/xlink}type": "extended"})
880
+ for refI, refObj in enumerate(taxonomyObj.get("references", [])):
881
+ referenceType = refObj.get("referenceType", "")
882
+ relatedIDs = refObj.get("relatedID", "")
883
+ xlinkLbl = xlinkLabelFormat.format("reference", refI+1)
884
+ refElt = addChild(elrElt, XbrlConst.qnLinkReference,
885
+ attributes={"{http://www.w3.org/1999/xlink}label": xlinkLbl,
886
+ "{http://www.w3.org/1999/xlink}role": referenceType,
887
+ "{http://www.w3.org/1999/xlink}type": "resource"})
888
+ for partObj in sorted(refObj.get("parts", []), key=lambda o:refObj.get("order", 0)):
889
+ name = partObj.get("name", "")
890
+ value = partObj.get("value", "")
891
+ partI = partObj.get("order", "")
892
+ addChild(refElt, qname(name, prefixNamespaces), text=value)
893
+ for relatedID in relatedIDs:
894
+ addChild(elrElt, XbrlConst.qnLinkLabelArc,
895
+ attributes={"{http://www.w3.org/1999/xlink}from": locXlinkLabel(elrElt, relatedID, f"reference[{refI}]/relatedID"),
896
+ "{http://www.w3.org/1999/xlink}to": xlinkLbl,
897
+ "{http://www.w3.org/1999/xlink}arcrole": XbrlConst.conceptReference,
898
+ "{http://www.w3.org/1999/xlink}type": "arc"})
899
+ lbEltArcroleRefs.add(XbrlConst.conceptReference)
900
+ addRoleRefs(lbElt, lbEltRoleRefs, lbEltArcroleRefs)
901
+
902
+ # discover linkbases
903
+ for lbElt in lbElts:
904
+ schemaDoc.linkbaseDiscover(lbElt)
905
+
906
+ # errors
907
+ for hrefNs, paths in sorted(hrefsNsWithoutPrefix.items(), key=lambda i:i[0]):
908
+ error("oimte:missingConceptRefPrefx",
909
+ "Namespace has no prefix %(namespace)s in %(paths)s",
910
+ modelObject=modelXbrl, namespace=hrefNs, paths=", ".join(paths))
911
+
912
+ # save schema files if specified
913
+ if (saveOIMTaxonomySchemaFiles or
914
+ modelXbrl.modelManager.formulaOptions.typedParameters(modelXbrl.prefixedNamespaces)
915
+ .get(SAVE_OIM_SCHEMA_FORULA_PARAMETER, ("",None))[1] not in (None, "", "false")):
916
+ schemaDoc.save(schemaDoc.filepath.replace(".json", "-json.xsd"))
917
+
918
+ return schemaDoc
919
+
920
+ except NotOIMException as ex:
921
+ _return = ex # not an OIM document
922
+ except Exception as ex:
923
+ _return = ex
924
+ if isinstance(ex, OIMException):
925
+ if ex.code and ex.message:
926
+ error(ex.code, ex.message, modelObject=modelXbrl, **ex.msgArgs)
927
+ else:
928
+ error("arelleOIMloader:error",
929
+ "Error while %(action)s, error %(errorType)s %(error)s\n traceback %(traceback)s",
930
+ modelObject=modelXbrl, action=currentAction, errorType=ex.__class__.__name__, error=ex,
931
+ traceback=traceback.format_tb(sys.exc_info()[2]))
932
+
933
+ global lastFilePath, lastFilePath
934
+ lastFilePath = None
935
+ lastFilePathIsOIM = False
936
+ return _return
937
+
938
+ def oimTaxonomyValidator(val, parameters):
939
+ if not isinstance(val.modelXbrl, XbrlDts): # if no OIM Taxonomy DTS give up
940
+ return
941
+ try:
942
+ validateDTS(val.modelXbrl)
943
+ except Exception as ex:
944
+ val.modelXbrl.error("arelleOIMloader:error",
945
+ "Error while validating, error %(errorType)s %(error)s\n traceback %(traceback)s",
946
+ modelObject=val.modelXbrl, errorType=ex.__class__.__name__, error=ex,
947
+ traceback=traceback.format_tb(sys.exc_info()[2]))
948
+
949
+ lastFilePath = None
950
+ lastFilePathIsOIM = False
951
+
952
+ def isOimTaxonomyLoadable(modelXbrl, mappedUri, normalizedUri, filepath, **kwargs):
953
+ global lastFilePath, lastFilePathIsOIM
954
+ lastFilePath = None
955
+ lastFilePathIsOIM = False
956
+ _ext = os.path.splitext(filepath)[1]
957
+ if _ext == ".json":
958
+ with io.open(filepath, 'rt', encoding='utf-8') as f:
959
+ _fileStart = f.read(4096)
960
+ if _fileStart and oimTaxonomyDocTypePattern.match(_fileStart):
961
+ lastFilePathIsOIM = True
962
+ lastFilePath = filepath
963
+ return lastFilePathIsOIM
964
+
965
+ def oimTaxonomyLoader(modelXbrl, mappedUri, filepath, *args, **kwargs):
966
+ if filepath != lastFilePath or not lastFilePathIsOIM:
967
+ return None # not an OIM file
968
+
969
+ cntlr = modelXbrl.modelManager.cntlr
970
+ cntlr.showStatus(_("Loading OIM taxonomy file: {0}").format(os.path.basename(filepath)))
971
+ doc = loadOIMTaxonomy(cntlr, modelXbrl.error, modelXbrl.warning, modelXbrl, filepath, mappedUri, **kwargs)
972
+ if doc is None:
973
+ return None # not an OIM file
974
+ return doc
975
+
976
+ def optionsExtender(parser, *args, **kwargs):
977
+ parser.add_option(SAVE_OIM_SCHEMA_CMDLINE_PARAMETER,
978
+ action="store_true",
979
+ dest="saveOIMTaxonomySchemaFiles",
980
+ help=_("Save each OIM taxonomy file an xsd named -json.xsd."))
981
+ def filingStart(self, options, *args, **kwargs):
982
+ global saveOIMTaxonomySchemaFiles
983
+ if options.saveOIMTaxonomySchemaFiles:
984
+ saveOIMTaxonomySchemaFiles = True
985
+
986
+ def oimTaxonomyLoaded(cntlr, options, xbrlDts, *args, **kwargs):
987
+ # index groupContents
988
+ xbrlDts.groupContents = defaultdict(OrderedSet)
989
+ for txmy in xbrlDts.taxonomies.values():
990
+ for grpCnts in txmy.groupContents:
991
+ for relName in getattr(grpCnts, "relatedNames", ()): # if object was invalid there are no attributes, e.g. bad QNames
992
+ xbrlDts.groupContents[grpCnts.groupName].add(relName)
993
+
994
+ def oimTaxonomyViews(cntlr, xbrlDts):
995
+ oimTaxonomyLoaded(cntlr, None, xbrlDts)
996
+ if isinstance(xbrlDts, XbrlDts):
997
+ initialViews = []
998
+ if getattr(xbrlDts, "facts", ()): # has instance facts
999
+ initialViews.append( (XbrlFact, cntlr.tabWinTopRt, "Report Facts") )
1000
+ initialViews.extend(((XbrlConcept, cntlr.tabWinBtm, "XBRL Concepts"),
1001
+ (XbrlGroup, cntlr.tabWinTopRt, "XBRL Groups"),
1002
+ (XbrlNetwork, cntlr.tabWinTopRt, "XBRL Networks"),
1003
+ (XbrlCube, cntlr.tabWinTopRt, "XBRL Cubes")
1004
+ ))
1005
+ initialViews = tuple(initialViews)
1006
+ additionalViews = ((XbrlAbstract, cntlr.tabWinBtm, "XBRL Abstracts"),
1007
+ (XbrlCubeType, cntlr.tabWinBtm, "XBRL Cube Types"),
1008
+ (XbrlDataType, cntlr.tabWinBtm, "XBRL Data Types"),
1009
+ (XbrlEntity, cntlr.tabWinBtm, "XBRL Entities"),
1010
+ (XbrlLabel, cntlr.tabWinBtm, "XBRL Labels"),
1011
+ (XbrlLabelType, cntlr.tabWinBtm, "XBRL Label Types"),
1012
+ (XbrlPropertyType, cntlr.tabWinBtm, "XBRL Property Types"),
1013
+ (XbrlReference, cntlr.tabWinBtm, "XBRL References"),
1014
+ (XbrlReferenceType, cntlr.tabWinBtm, "XBRL Reference Types"),
1015
+ (XbrlRelationshipType, cntlr.tabWinBtm, "XBRL Relationship Types"),
1016
+ (XbrlTransform, cntlr.tabWinBtm, "XBRL Transforms"),
1017
+ (XbrlUnit, cntlr.tabWinBtm, "XBRL Units"),)
1018
+ for view in initialViews:
1019
+ viewXbrlTxmyObj(xbrlDts, *view, additionalViews)
1020
+ return True # block ordinary taxonomy views
1021
+ return False
1022
+
1023
+ __pluginInfo__ = {
1024
+ 'name': 'OIM Taxonomy',
1025
+ 'version': '1.2',
1026
+ 'description': "This plug-in implements XBRL taxonomy objects loaded from JSON.",
1027
+ 'license': 'Apache-2',
1028
+ 'author': authorLabel,
1029
+ 'copyright': copyrightLabel,
1030
+ # classes of mount points (required)
1031
+ 'CntlrCmdLine.Options': optionsExtender,
1032
+ 'CntlrCmdLine.Filing.Start': filingStart,
1033
+ 'CntlrCmdLine.Xbrl.Loaded': oimTaxonomyLoaded,
1034
+ 'CntlrWinMain.Xbrl.Views': oimTaxonomyViews,
1035
+ 'ModelDocument.IsPullLoadable': isOimTaxonomyLoadable,
1036
+ 'ModelDocument.PullLoader': oimTaxonomyLoader,
1037
+ 'Validate.XBRL.Start': oimTaxonomyValidator
1038
+ }