xbrlkit 0.4.0__tar.gz → 0.4.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/PKG-INFO +65 -7
  2. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/README.md +64 -6
  3. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/pyproject.toml +1 -1
  4. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/tests/test_oim.py +17 -0
  5. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/tests/test_tavi.py +114 -0
  6. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/tests/test_text_ixbrl.py +77 -5
  7. xbrlkit-0.4.2/xbrlkit/__init__.py +28 -0
  8. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/model.py +4 -0
  9. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/serialize/_values.py +21 -7
  10. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/serialize/oim.py +5 -5
  11. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/serialize/tavi.py +62 -17
  12. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/text/ixbrl.py +40 -11
  13. xbrlkit-0.4.0/xbrlkit/__init__.py +0 -24
  14. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/.gitignore +0 -0
  15. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/LICENSE +0 -0
  16. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/pytest.ini +0 -0
  17. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/ruff.toml +0 -0
  18. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/tests/__init__.py +0 -0
  19. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/tests/conftest.py +0 -0
  20. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/tests/test_arelle_cache.py +0 -0
  21. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/tests/test_config.py +0 -0
  22. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/tests/test_edgar.py +0 -0
  23. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/tests/test_efts.py +0 -0
  24. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/tests/test_lpg.py +0 -0
  25. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/tests/test_parity.py +0 -0
  26. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/tests/test_parse.py +0 -0
  27. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/tests/test_query.py +0 -0
  28. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/tests/test_serialize.py +0 -0
  29. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/tests/test_text_narrative.py +0 -0
  30. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/tests/test_text_parts.py +0 -0
  31. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/tests/test_text_tables.py +0 -0
  32. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/__init__.py +0 -0
  33. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/__init__.py +0 -0
  34. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/conf/README.md +0 -0
  35. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/conf/extractTestcase.sh +0 -0
  36. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/conf/extractTestcase.xsl +0 -0
  37. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/conf/runIxtSecTests.sh +0 -0
  38. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/conf/saxon9.jar +0 -0
  39. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/conf/testcase.xml +0 -0
  40. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/conf/tests.xml +0 -0
  41. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/conf/tests.xsd +0 -0
  42. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/text2num.py +0 -0
  43. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/transformationRegistry/registry/ixt-sec-boolballotbox.xml +0 -0
  44. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/transformationRegistry/registry/ixt-sec-countrynameen.xml +0 -0
  45. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/transformationRegistry/registry/ixt-sec-datequarterend.xml +0 -0
  46. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/transformationRegistry/registry/ixt-sec-durday.xml +0 -0
  47. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/transformationRegistry/registry/ixt-sec-durhour.xml +0 -0
  48. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/transformationRegistry/registry/ixt-sec-durmonth.xml +0 -0
  49. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/transformationRegistry/registry/ixt-sec-durweek.xml +0 -0
  50. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/transformationRegistry/registry/ixt-sec-durwordsen.xml +0 -0
  51. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/transformationRegistry/registry/ixt-sec-duryear.xml +0 -0
  52. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/transformationRegistry/registry/ixt-sec-edgarprovcountryen.xml +0 -0
  53. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/transformationRegistry/registry/ixt-sec-entityfilercategoryen.xml +0 -0
  54. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/transformationRegistry/registry/ixt-sec-exchnameen.xml +0 -0
  55. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/transformationRegistry/registry/ixt-sec-numwordsen.xml +0 -0
  56. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/transformationRegistry/registry/ixt-sec-stateprovnameen.xml +0 -0
  57. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/transformationRegistry/registry/ixt-sec-yesnoballotbox.xml +0 -0
  58. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/transformationRegistry/registry/transform-registry.xml +0 -0
  59. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/transformationRegistry/schema/inlinexbrl-sec-transformation.xsd +0 -0
  60. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/ontology/v1/context.jsonld +0 -0
  61. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/ontology/v1/shapes.ttl +0 -0
  62. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/cli.py +0 -0
  63. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/config.py +0 -0
  64. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/edgar/__init__.py +0 -0
  65. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/edgar/client.py +0 -0
  66. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/edgar/download.py +0 -0
  67. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/edgar/efts.py +0 -0
  68. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/edgar/rate_limit.py +0 -0
  69. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/namespaces.py +0 -0
  70. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/parse/__init__.py +0 -0
  71. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/parse/arelle_load.py +0 -0
  72. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/parse/cache.py +0 -0
  73. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/parse/ids.py +0 -0
  74. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/parse/to_model.py +0 -0
  75. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/py.typed +0 -0
  76. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/query.py +0 -0
  77. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/schema/__init__.py +0 -0
  78. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/serialize/__init__.py +0 -0
  79. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/serialize/_kernel/__init__.py +0 -0
  80. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/serialize/_kernel/bundle.py +0 -0
  81. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/serialize/_kernel/context.py +0 -0
  82. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/serialize/_kernel/holon.py +0 -0
  83. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/serialize/_kernel/jsonld.py +0 -0
  84. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/serialize/classify.py +0 -0
  85. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/serialize/graph.py +0 -0
  86. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/serialize/holon.py +0 -0
  87. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/serialize/lpg.py +0 -0
  88. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/text/__init__.py +0 -0
  89. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/text/narrative.py +0 -0
  90. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/text/parts.py +0 -0
  91. {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/text/tables.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: xbrlkit
3
- Version: 0.4.0
3
+ Version: 0.4.2
4
4
  Summary: Work with XBRL filings above Arelle: one parse, portable models
5
5
  Project-URL: Repository, https://github.com/RoboFinSystems/xbrlkit
6
6
  Project-URL: Homepage, https://github.com/RoboFinSystems/xbrlkit#readme
@@ -60,12 +60,16 @@ it **once** into a neutral typed model, and project that model into whichever
60
60
  portable representation you need.
61
61
 
62
62
  ```
63
- EDGAR ──▶ Arelle ──▶ XbrlModel ──┬──▶ holon.jsonld (RDF / JSON-LD)
64
- ├──▶ Tavi (compiled model)
65
- ├──▶ xBRL-JSON (OIM)
66
- └──▶ …
63
+ EDGAR ──▶ Arelle ──▶ XbrlModel ──┬──▶ holon.jsonld (RDF / JSON-LD)
64
+ │ ├──▶ Tavi (compiled model)
65
+ │ ├──▶ xBRL-JSON (OIM)
66
+ │ └──▶ property graph (parquet, .lbug)
67
+ └──▶ primary HTML ──▶ xbrlkit.text ──▶ sections (text blocks, Items, tables)
67
68
  ```
68
69
 
70
+ Four projections hang off the model. A fifth surface, the filing's text, reads the
71
+ primary HTML document directly and needs neither Arelle nor the network.
72
+
69
73
  Arelle stays the parser — nobody should reimplement DTS resolution. What it does
70
74
  not give you is anything ergonomic to *hold*: `ModelXbrl` is a large mutable
71
75
  object graph tied to a controller you have to close. `XbrlModel` is the answer to
@@ -141,7 +145,7 @@ network — and returns its text as sections:
141
145
 
142
146
  | Parser | Sections | Notes |
143
147
  | --- | --- | --- |
144
- | `iXBRLParser` | every inline-XBRL text block (notes, policies, tables), with the XBRL element names it contains | `ix:continuation` chains resolved; nested continuations and nested text blocks included |
148
+ | `iXBRLParser` | every inline-XBRL text block (notes, policies, tables), with the XBRL element names it contains | `ix:continuation` chains resolved; nested continuations and nested text blocks included; a concept tagged more than once is one section holding every occurrence; `ix:exclude` page furniture dropped |
145
149
  | `NarrativeExtractor` | the 10-K / 10-Q Items — Business, Risk Factors, Cybersecurity, Properties, MD&A, Market Risk | table-of-contents rows and cross-references rejected; a 10-Q's Part I and Part II Items kept apart |
146
150
 
147
151
  Both render HTML tables as markdown pipe tables and split a long section into
@@ -207,7 +211,14 @@ SEC_GOV_USER_AGENT="Your Name your@email.com"
207
211
  # Build a holon.jsonld from a specific filing (-> ./output/)
208
212
  xbrlkit build --cik 320193 --accno 0000320193-23-000106
209
213
 
210
- # Fetch the latest filing for a ticker (-> ./output/)
214
+ # The other projections: Tavi (plus its .tavi.gaps.json sidecar), xBRL-JSON,
215
+ # the property graph (needs the lpg extra), or every one of them
216
+ xbrlkit build --cik 320193 --accno 0000320193-23-000106 --format tavi
217
+ xbrlkit build --cik 320193 --accno 0000320193-23-000106 --format oim
218
+ xbrlkit build --cik 320193 --accno 0000320193-23-000106 --format lpg
219
+ xbrlkit build --cik 320193 --accno 0000320193-23-000106 --format all
220
+
221
+ # Fetch the latest filing for a ticker (-> ./output/); --form and --n filter
211
222
  xbrlkit fetch --ticker NVDA
212
223
 
213
224
  # Query consolidated facts in a built holon (in-memory SPARQL)
@@ -217,6 +228,31 @@ xbrlkit query --in output/0000320193-23-000106.holon.jsonld --element us-gaap:As
217
228
  From a source checkout, `just` wraps the same CLI as a shorthand:
218
229
  `just build 320193 0000320193-23-000106` and `just fetch NVDA`.
219
230
 
231
+ ## EDGAR
232
+
233
+ `xbrlkit.edgar` is the fetch layer the CLI uses, exposed for hosts that discover
234
+ and download filings themselves: synchronous `requests`, local-filesystem output,
235
+ and EDGAR's two throttle signatures — a 429, and an empty 200 — ridden out with
236
+ a bounded wait-and-retry (`EdgarThrottled` when the budget is spent).
237
+
238
+ | | |
239
+ | --- | --- |
240
+ | `EdgarClient` | ticker → CIK, a company's filing list (`list_filings`, by form), `company_info`, one filing by accession (`get_filing_ref`) |
241
+ | `EftsClient` / `query_efts` | bulk discovery through EDGAR full-text search: by form, year or quarter, across every filer |
242
+ | `download_filing` / `fetch` | the XBRL zip for one accession, unpacked to a directory |
243
+
244
+ ```python
245
+ from pathlib import Path
246
+ from xbrlkit.edgar import EdgarClient, download_filing
247
+
248
+ client = EdgarClient()
249
+ cik = client.ticker_to_cik("MMM")
250
+ latest = client.list_filings(cik, forms=["10-K"])[0]
251
+ package = download_filing(client, cik, latest.accession, Path("data")) # the Arelle load target
252
+ ```
253
+
254
+ The SEC User-Agent is required here as everywhere (see [SEC User-Agent](#sec-user-agent)).
255
+
220
256
  ## Arelle cache
221
257
 
222
258
  Arelle resolves a filing's DTS by fetching every schema and linkbase it imports —
@@ -242,6 +278,28 @@ xbrlkit cache extract --bundle schemas.tar.gz # seed a container's cache at buil
242
278
  network; a miss is then an error, not a fetch. A host that builds its own Arelle
243
279
  controller gets the same policy from `xbrlkit.parse.configure_webcache(cntlr, cache_dir)`.
244
280
 
281
+ ## Where it runs
282
+
283
+ **RoboSystems.** The platform's SEC pipeline is built on this package: filings are
284
+ parsed with `xbrlkit.parse` (the platform's own Arelle controller, with
285
+ `register_sec_transforms` and the cache policy from `configure_webcache`), projected
286
+ with `to_holon`, `to_tavi_report` and the property-graph tables, the shared `sec`
287
+ graph is declared from `xbrlkit.schema`, and the full-text index behind its document
288
+ search is built from `xbrlkit.text`.
289
+
290
+ **Filing Ladder.** The [Filing Ladder](https://github.com/HarbingerFinLab/filing-ladder)
291
+ benchmark — one filing handed to the same language model in every representation —
292
+ built its 26-filing corpus of 2024–2025 10-Ks and 10-Qs with this package: the Tavi
293
+ compiled model, the holon, the per-filing property graph, and both text parsers. Each
294
+ projection is a rung of the ladder, so the
295
+ [v0.1 results](https://github.com/HarbingerFinLab/filing-ladder/blob/main/results/v0.1-sonnet-5/README.md)
296
+ are also a measurement of what a model can do with each of these outputs. Before that
297
+ run, every text-block section the parsers produce was checked against the filing's own
298
+ text-block facts as Arelle resolves them, on all 26 filings, and the property-graph
299
+ projection was checked row for row against the platform's processor. The two defects
300
+ those checks found in the text layer were fixed in 0.4.1 and are disclosed in the
301
+ benchmark's protocol.
302
+
245
303
  ## View & explore
246
304
 
247
305
  Built holons render in the **RoboSystems Holon Viewer** — a browser-based reader
@@ -7,12 +7,16 @@ it **once** into a neutral typed model, and project that model into whichever
7
7
  portable representation you need.
8
8
 
9
9
  ```
10
- EDGAR ──▶ Arelle ──▶ XbrlModel ──┬──▶ holon.jsonld (RDF / JSON-LD)
11
- ├──▶ Tavi (compiled model)
12
- ├──▶ xBRL-JSON (OIM)
13
- └──▶ …
10
+ EDGAR ──▶ Arelle ──▶ XbrlModel ──┬──▶ holon.jsonld (RDF / JSON-LD)
11
+ │ ├──▶ Tavi (compiled model)
12
+ │ ├──▶ xBRL-JSON (OIM)
13
+ │ └──▶ property graph (parquet, .lbug)
14
+ └──▶ primary HTML ──▶ xbrlkit.text ──▶ sections (text blocks, Items, tables)
14
15
  ```
15
16
 
17
+ Four projections hang off the model. A fifth surface, the filing's text, reads the
18
+ primary HTML document directly and needs neither Arelle nor the network.
19
+
16
20
  Arelle stays the parser — nobody should reimplement DTS resolution. What it does
17
21
  not give you is anything ergonomic to *hold*: `ModelXbrl` is a large mutable
18
22
  object graph tied to a controller you have to close. `XbrlModel` is the answer to
@@ -88,7 +92,7 @@ network — and returns its text as sections:
88
92
 
89
93
  | Parser | Sections | Notes |
90
94
  | --- | --- | --- |
91
- | `iXBRLParser` | every inline-XBRL text block (notes, policies, tables), with the XBRL element names it contains | `ix:continuation` chains resolved; nested continuations and nested text blocks included |
95
+ | `iXBRLParser` | every inline-XBRL text block (notes, policies, tables), with the XBRL element names it contains | `ix:continuation` chains resolved; nested continuations and nested text blocks included; a concept tagged more than once is one section holding every occurrence; `ix:exclude` page furniture dropped |
92
96
  | `NarrativeExtractor` | the 10-K / 10-Q Items — Business, Risk Factors, Cybersecurity, Properties, MD&A, Market Risk | table-of-contents rows and cross-references rejected; a 10-Q's Part I and Part II Items kept apart |
93
97
 
94
98
  Both render HTML tables as markdown pipe tables and split a long section into
@@ -154,7 +158,14 @@ SEC_GOV_USER_AGENT="Your Name your@email.com"
154
158
  # Build a holon.jsonld from a specific filing (-> ./output/)
155
159
  xbrlkit build --cik 320193 --accno 0000320193-23-000106
156
160
 
157
- # Fetch the latest filing for a ticker (-> ./output/)
161
+ # The other projections: Tavi (plus its .tavi.gaps.json sidecar), xBRL-JSON,
162
+ # the property graph (needs the lpg extra), or every one of them
163
+ xbrlkit build --cik 320193 --accno 0000320193-23-000106 --format tavi
164
+ xbrlkit build --cik 320193 --accno 0000320193-23-000106 --format oim
165
+ xbrlkit build --cik 320193 --accno 0000320193-23-000106 --format lpg
166
+ xbrlkit build --cik 320193 --accno 0000320193-23-000106 --format all
167
+
168
+ # Fetch the latest filing for a ticker (-> ./output/); --form and --n filter
158
169
  xbrlkit fetch --ticker NVDA
159
170
 
160
171
  # Query consolidated facts in a built holon (in-memory SPARQL)
@@ -164,6 +175,31 @@ xbrlkit query --in output/0000320193-23-000106.holon.jsonld --element us-gaap:As
164
175
  From a source checkout, `just` wraps the same CLI as a shorthand:
165
176
  `just build 320193 0000320193-23-000106` and `just fetch NVDA`.
166
177
 
178
+ ## EDGAR
179
+
180
+ `xbrlkit.edgar` is the fetch layer the CLI uses, exposed for hosts that discover
181
+ and download filings themselves: synchronous `requests`, local-filesystem output,
182
+ and EDGAR's two throttle signatures — a 429, and an empty 200 — ridden out with
183
+ a bounded wait-and-retry (`EdgarThrottled` when the budget is spent).
184
+
185
+ | | |
186
+ | --- | --- |
187
+ | `EdgarClient` | ticker → CIK, a company's filing list (`list_filings`, by form), `company_info`, one filing by accession (`get_filing_ref`) |
188
+ | `EftsClient` / `query_efts` | bulk discovery through EDGAR full-text search: by form, year or quarter, across every filer |
189
+ | `download_filing` / `fetch` | the XBRL zip for one accession, unpacked to a directory |
190
+
191
+ ```python
192
+ from pathlib import Path
193
+ from xbrlkit.edgar import EdgarClient, download_filing
194
+
195
+ client = EdgarClient()
196
+ cik = client.ticker_to_cik("MMM")
197
+ latest = client.list_filings(cik, forms=["10-K"])[0]
198
+ package = download_filing(client, cik, latest.accession, Path("data")) # the Arelle load target
199
+ ```
200
+
201
+ The SEC User-Agent is required here as everywhere (see [SEC User-Agent](#sec-user-agent)).
202
+
167
203
  ## Arelle cache
168
204
 
169
205
  Arelle resolves a filing's DTS by fetching every schema and linkbase it imports —
@@ -189,6 +225,28 @@ xbrlkit cache extract --bundle schemas.tar.gz # seed a container's cache at buil
189
225
  network; a miss is then an error, not a fetch. A host that builds its own Arelle
190
226
  controller gets the same policy from `xbrlkit.parse.configure_webcache(cntlr, cache_dir)`.
191
227
 
228
+ ## Where it runs
229
+
230
+ **RoboSystems.** The platform's SEC pipeline is built on this package: filings are
231
+ parsed with `xbrlkit.parse` (the platform's own Arelle controller, with
232
+ `register_sec_transforms` and the cache policy from `configure_webcache`), projected
233
+ with `to_holon`, `to_tavi_report` and the property-graph tables, the shared `sec`
234
+ graph is declared from `xbrlkit.schema`, and the full-text index behind its document
235
+ search is built from `xbrlkit.text`.
236
+
237
+ **Filing Ladder.** The [Filing Ladder](https://github.com/HarbingerFinLab/filing-ladder)
238
+ benchmark — one filing handed to the same language model in every representation —
239
+ built its 26-filing corpus of 2024–2025 10-Ks and 10-Qs with this package: the Tavi
240
+ compiled model, the holon, the per-filing property graph, and both text parsers. Each
241
+ projection is a rung of the ladder, so the
242
+ [v0.1 results](https://github.com/HarbingerFinLab/filing-ladder/blob/main/results/v0.1-sonnet-5/README.md)
243
+ are also a measurement of what a model can do with each of these outputs. Before that
244
+ run, every text-block section the parsers produce was checked against the filing's own
245
+ text-block facts as Arelle resolves them, on all 26 filings, and the property-graph
246
+ projection was checked row for row against the platform's processor. The two defects
247
+ those checks found in the text layer were fixed in 0.4.1 and are disclosed in the
248
+ benchmark's protocol.
249
+
192
250
  ## View & explore
193
251
 
194
252
  Built holons render in the **RoboSystems Holon Viewer** — a browser-based reader
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "xbrlkit"
3
- version = "0.4.0"
3
+ version = "0.4.2"
4
4
  description = "Work with XBRL filings above Arelle: one parse, portable models"
5
5
  readme = "README.md"
6
6
  requires-python = ">=3.10"
@@ -153,6 +153,23 @@ def test_document_envelope() -> None:
153
153
  assert info["namespaces"]["cik"] == "http://www.sec.gov/CIK"
154
154
 
155
155
 
156
+ def test_non_sec_entity_binds_its_own_scheme() -> None:
157
+ """An entity under any scheme but the SEC's is ``entity:<id>``, scheme bound."""
158
+ model = _model().model_copy(
159
+ update={
160
+ "entity": EntityIdentity(cik="ent_01K3ZQ", scheme="http://robosystems.ai/entity")
161
+ }
162
+ )
163
+ document = to_oim_document(model)
164
+ namespaces = document["documentInfo"]["namespaces"]
165
+ assert namespaces["entity"] == "http://robosystems.ai/entity"
166
+ assert "cik" not in namespaces
167
+ assert all(
168
+ fact["dimensions"]["entity"] == "entity:ent_01K3ZQ"
169
+ for fact in document["facts"].values()
170
+ )
171
+
172
+
156
173
  def test_instant_period_is_the_exclusive_next_midnight() -> None:
157
174
  """The close of 2024-12-31 is written as the instant after it."""
158
175
  assert _by_concept("us-gaap:Assets")["dimensions"]["period"] == "2025-01-01T00:00:00"
@@ -407,6 +407,85 @@ def test_entity_sqname_keeps_the_scheme() -> None:
407
407
  assert _document()["xbrlModel"]["entities"] == [{"name": "cik:0001234567"}]
408
408
 
409
409
 
410
+ def _ledger_model() -> XbrlModel:
411
+ """The same model re-homed on an entity that is not an EDGAR filer."""
412
+ return _model().model_copy(
413
+ update={
414
+ "entity": EntityIdentity(
415
+ cik="ent_01K3ZQ", scheme="http://robosystems.ai/entity", name="Acme LLC"
416
+ )
417
+ }
418
+ )
419
+
420
+
421
+ def test_non_sec_entity_binds_its_own_scheme_under_entity() -> None:
422
+ """Section 8.1 again: the prefix follows the scheme, not the SEC.
423
+
424
+ A model built from a ledger's own report has no CIK. Its entity is still
425
+ scheme + identifier — written ``entity:<id>`` under the scheme it declares,
426
+ with no ``cik`` binding at all.
427
+ """
428
+ document, _ = to_tavi_report(_ledger_model())
429
+ namespaces = document["documentInfo"]["namespaces"]
430
+ assert namespaces["entity"] == "http://robosystems.ai/entity"
431
+ assert "cik" not in namespaces
432
+ assert document["xbrlModel"]["entities"] == [{"name": "entity:ent_01K3ZQ"}]
433
+ assert all(
434
+ fact["factDimensions"]["xbrl:entity"] == "entity:ent_01K3ZQ"
435
+ for fact in document["xbrlModel"]["facts"]
436
+ )
437
+
438
+
439
+ def test_entity_name_is_a_label_on_the_entity() -> None:
440
+ """The entity's name is a label object pointing at the entity (5.14).
441
+
442
+ A reader holding the compiled model and nothing else — no ``dei`` facts, no
443
+ EDGAR header — still needs a name to put on the report.
444
+ """
445
+ labels = [
446
+ entry
447
+ for entry in _document()["xbrlModel"]["labels"]
448
+ if entry["forObject"] == "cik:0001234567"
449
+ ]
450
+ assert labels == [
451
+ {
452
+ "forObject": "cik:0001234567",
453
+ "labelType": "xbrl:label",
454
+ "value": "Acme Corp",
455
+ "language": "en-US",
456
+ }
457
+ ]
458
+
459
+
460
+ def test_entity_without_a_name_has_no_label() -> None:
461
+ model = _model().model_copy(update={"entity": EntityIdentity(cik="0001234567")})
462
+ document, _ = to_tavi_report(model)
463
+ assert not [
464
+ entry
465
+ for entry in document["xbrlModel"]["labels"]
466
+ if entry["forObject"] == "cik:0001234567"
467
+ ]
468
+
469
+
470
+ def test_description_reads_the_filing_unless_one_is_supplied() -> None:
471
+ assert (
472
+ _document()["documentInfo"]["description"]
473
+ == "10-K 0000000000-24-000001 (CIK 0001234567) projected from XBRL by xbrlkit"
474
+ )
475
+ document, _ = to_tavi_report(
476
+ _ledger_model(), description="RoboLedger report r1 g2 (Acme LLC)"
477
+ )
478
+ assert document["documentInfo"]["description"] == (
479
+ "RoboLedger report r1 g2 (Acme LLC)"
480
+ )
481
+ assert (
482
+ json.loads(to_tavi(_ledger_model(), description="custom"))["documentInfo"][
483
+ "description"
484
+ ]
485
+ == "custom"
486
+ )
487
+
488
+
410
489
  def test_calculation_relationship_carries_weight_only() -> None:
411
490
  """Section 14.3.1: weight is required; reconciliation is a flag, not asserted."""
412
491
  networks = _document()["xbrlModel"]["networks"]
@@ -463,6 +542,41 @@ def test_group_definition_is_a_label_with_the_filing_language() -> None:
463
542
  assert all("language" in entry for entry in model["labels"])
464
543
 
465
544
 
545
+ def test_network_documentation_is_a_second_label_on_the_group() -> None:
546
+ """A role's longer reading rides as an ``xbrl:documentation`` label.
547
+
548
+ A producer whose definition is a composed sort key keeps the role's own
549
+ name in ``documentation``; a network without one adds no label.
550
+ """
551
+ model = _model()
552
+ networks = [
553
+ n.model_copy(update={"documentation": "Consolidated Balance Sheets"})
554
+ if n.role_uri == "http://example.com/role/BalanceSheet"
555
+ else n
556
+ for n in model.networks
557
+ ]
558
+ document, _ = to_tavi_report(model.model_copy(update={"networks": networks}))
559
+ xbrl_model = document["xbrlModel"]
560
+ group_name = next(
561
+ g["name"]
562
+ for g in xbrl_model["groups"]
563
+ if g["groupURI"] == "http://example.com/role/BalanceSheet"
564
+ )
565
+ labels = {
566
+ entry["labelType"]: entry["value"]
567
+ for entry in xbrl_model["labels"]
568
+ if entry["forObject"] == group_name
569
+ }
570
+ assert labels == {
571
+ "xbrl:label": "Balance Sheet",
572
+ "xbrl:documentation": "Consolidated Balance Sheets",
573
+ }
574
+ plain = _document()["xbrlModel"]
575
+ assert not [
576
+ entry for entry in plain["labels"] if entry["labelType"] == "xbrl:documentation"
577
+ ]
578
+
579
+
466
580
  def test_shares_map_to_the_accounting_module() -> None:
467
581
  """No core datatype types a share count; the xbrla module does."""
468
582
  document, gaps = to_tavi_report(_model())
@@ -5,7 +5,7 @@ Covers:
5
5
  - Continuation chain resolution, nested continuations included
6
6
  - Element extraction from nested ix:nonFraction tags
7
7
  - Label derivation from element qnames
8
- - Filtering (DEI/ECD skip, min word count, dedup)
8
+ - Filtering (DEI/ECD skip, min word count), repeated tags, ix:exclude
9
9
  - Splitting a long section into parts
10
10
  """
11
11
 
@@ -248,20 +248,92 @@ class TestiXBRLParser:
248
248
  """
249
249
  assert iXBRLParser().parse(html) == []
250
250
 
251
- def test_deduplicates_same_element(self):
251
+ def test_same_text_tagged_twice_counts_once(self):
252
+ """The same table tagged twice with different markup (Smucker's
253
+ intangibles-acquired table) is one occurrence, not two."""
252
254
  html = f"""
253
255
  <html><body>
254
256
  <ix:nonNumeric contextRef="c-1" name="us-gaap:RevenueTextBlock" id="f-1">
255
- First occurrence {_FILLER}
257
+ <p>Same occurrence {_FILLER}</p>
256
258
  </ix:nonNumeric>
257
259
  <ix:nonNumeric contextRef="c-2" name="us-gaap:RevenueTextBlock" id="f-2">
258
- Second occurrence {_FILLER}
260
+ <div><span>Same occurrence</span> {_FILLER}</div>
259
261
  </ix:nonNumeric>
260
262
  </body></html>
261
263
  """
262
264
  sections = iXBRLParser().parse(html)
263
265
  assert len(sections) == 1
264
- assert "First occurrence" in sections[0].content
266
+ assert sections[0].content.count("Same occurrence") == 1
267
+
268
+ def test_merges_distinct_occurrences_of_a_concept(self):
269
+ """A concept tagged more than once — Microsoft's purchase-price table for
270
+ each of two acquisitions, loanDepot's servicing-assets roll-forward per
271
+ period — is one section holding every occurrence in document order.
272
+ Keeping the first alone lost the second table."""
273
+ html = f"""
274
+ <html><body>
275
+ <ix:nonNumeric contextRef="c-1" name="us-gaap:ScheduleOfBusinessAcquisitionsByAcquisitionTextBlock" id="f-1">
276
+ <p>Nuance purchase price {_FILLER}
277
+ <ix:nonFraction name="us-gaap:Goodwill" contextRef="c-1">16326</ix:nonFraction></p>
278
+ </ix:nonNumeric>
279
+ <p>Other text between the two tables.</p>
280
+ <ix:nonNumeric contextRef="c-2" name="us-gaap:ScheduleOfBusinessAcquisitionsByAcquisitionTextBlock" id="f-2">
281
+ <p>Activision purchase price {_FILLER}
282
+ <ix:nonFraction name="us-gaap:BusinessCombinationConsiderationTransferred1" contextRef="c-2">61800</ix:nonFraction></p>
283
+ </ix:nonNumeric>
284
+ </body></html>
285
+ """
286
+ sections = iXBRLParser().parse(html)
287
+ assert len(sections) == 1
288
+ s = sections[0]
289
+ assert s.content.index("Nuance purchase price") < s.content.index(
290
+ "Activision purchase price"
291
+ )
292
+ assert "Other text between" not in s.content
293
+ assert s.xbrl_elements == [
294
+ "us-gaap:BusinessCombinationConsiderationTransferred1",
295
+ "us-gaap:Goodwill",
296
+ ]
297
+
298
+ def test_nested_block_of_the_same_name_is_not_repeated(self):
299
+ html = f"""
300
+ <html><body>
301
+ <ix:nonNumeric contextRef="c-1" name="us-gaap:DebtDisclosureTextBlock" id="f-1">
302
+ <p>Outer debt note {_FILLER}</p>
303
+ <ix:nonNumeric contextRef="c-2" name="us-gaap:DebtDisclosureTextBlock" id="f-2">
304
+ <p>Inner repeat {_FILLER}</p>
305
+ </ix:nonNumeric>
306
+ </ix:nonNumeric>
307
+ </body></html>
308
+ """
309
+ sections = iXBRLParser().parse(html)
310
+ assert len(sections) == 1
311
+ assert sections[0].content.count("Inner repeat") == 1
312
+
313
+ def test_drops_ix_exclude_content(self):
314
+ """``ix:exclude`` marks content inside a tagged block that is not part
315
+ of the fact — the page header Oracle and Microsoft place inside a policy
316
+ (32 and 50 per filing). Arelle drops it; so must the section."""
317
+ html = f"""
318
+ <html><body>
319
+ <ix:nonNumeric contextRef="c-1" name="us-gaap:IncomeTaxPolicyTextBlock" id="f-1" continuedAt="f-1-a">
320
+ <p>Income taxes are accounted for under ASC 740. {_FILLER}</p>
321
+ <ix:exclude><div><p>80</p><p>Table of Contents</p><p>Index to Financial Statements</p></div></ix:exclude>
322
+ <p>The second step measures the benefit.</p>
323
+ </ix:nonNumeric>
324
+ <ix:continuation id="f-1-a">
325
+ <ix:exclude><p>81 Table of Contents</p></ix:exclude>
326
+ <p>Continued policy text after the page break.</p>
327
+ </ix:continuation>
328
+ </body></html>
329
+ """
330
+ sections = iXBRLParser().parse(html)
331
+ assert len(sections) == 1
332
+ content = sections[0].content
333
+ assert "Table of Contents" not in content
334
+ assert "Index to Financial Statements" not in content
335
+ assert "The second step measures the benefit." in content
336
+ assert "Continued policy text after the page break." in content
265
337
 
266
338
  def test_multiple_sections(self):
267
339
  html = f"""
@@ -0,0 +1,28 @@
1
+ """xbrlkit — work with XBRL filings above Arelle: one parse, portable models.
2
+
3
+ Fetch a filing from EDGAR (:mod:`xbrlkit.edgar`), parse it once into the
4
+ neutral :class:`XbrlModel` (:mod:`xbrlkit.parse`), and project that model
5
+ into whichever representation you need (:mod:`xbrlkit.serialize`): the
6
+ ``holon.jsonld`` RDF/JSON-LD document, the Project Tavi compiled model,
7
+ xBRL-JSON, or the property-graph tables and a single-filing LadybugDB file.
8
+ :mod:`xbrlkit.text` reads the filing's primary HTML into sections without
9
+ Arelle at all; :mod:`xbrlkit.query` runs SPARQL over a built holon.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ from importlib.metadata import PackageNotFoundError, version
15
+
16
+ from .model import XbrlModel
17
+
18
+
19
+ def _get_version() -> str:
20
+ try:
21
+ return version("xbrlkit")
22
+ except PackageNotFoundError:
23
+ return "0.0.0+development"
24
+
25
+
26
+ __version__ = _get_version()
27
+
28
+ __all__ = ("XbrlModel", "__version__")
@@ -264,6 +264,10 @@ class Network(BaseModel):
264
264
 
265
265
  role_uri: str
266
266
  definition: str | None = None
267
+ # A second, longer reading of the role — XBRL's documentation label role.
268
+ # A producer whose definition is a composed sort key ("0001 - Statement -
269
+ # Balance Sheet") keeps the role's own name here, so it is not lost.
270
+ documentation: str | None = None
267
271
  kind: NetworkKind
268
272
  arcs: list[Arc] = Field(default_factory=list)
269
273
  # The ``id`` of the role's ``<link:roleType>`` in the filer's schema, when
@@ -3,27 +3,41 @@
3
3
  xBRL-JSON (REC 2021) and Tavi (PWD 2026-09-01) are two serialisations of the
4
4
  same Open Information Model, and the literal forms below are where they agree:
5
5
  a period is an ISO 8601 interval of ``xs:dateTime`` values with an exclusive
6
- end, a language tag is lower case, and the SEC entity scheme is written with
7
- the ``cik`` prefix. Keeping them in one place means a fact's period or entity
8
- reads identically in both projections of one filing — which is also what lets
9
- one be checked against the other.
6
+ end, a language tag is lower case, and an entity is written scheme-first: the
7
+ SEC scheme under the ``cik`` prefix, any other under ``entity``. Keeping them
8
+ in one place means a fact's period or entity reads identically in both
9
+ projections of one filing — which is also what lets one be checked against
10
+ the other.
10
11
  """
11
12
 
12
13
  from __future__ import annotations
13
14
 
14
15
  from datetime import date, timedelta
15
16
 
16
- from ..model import Period
17
+ from ..model import EntityIdentity, Period
17
18
 
18
19
  # The SEC's entity scheme, bound to the prefix Arelle uses for it. An entity is
19
20
  # identified by scheme + identifier, so the scheme is the namespace and the
20
21
  # CIK the local name: ``cik:0000066740``.
21
22
  CIK_PREFIX = "cik"
22
23
  CIK_SCHEME = "http://www.sec.gov/CIK"
24
+ # Any other scheme binds under a neutral prefix. A model that did not come from
25
+ # an EDGAR filing — a ledger's own report, say — still identifies its entity by
26
+ # scheme + identifier, and writes it ``entity:<identifier>``.
27
+ ENTITY_PREFIX = "entity"
23
28
 
24
29
 
25
- def entity_sqname(cik: str) -> str:
26
- return f"{CIK_PREFIX}:{cik}"
30
+ def entity_prefix(entity: EntityIdentity) -> tuple[str, str]:
31
+ """The ``(prefix, scheme)`` pair an entity's SQName is bound under."""
32
+ if entity.scheme == CIK_SCHEME:
33
+ return CIK_PREFIX, CIK_SCHEME
34
+ return ENTITY_PREFIX, entity.scheme
35
+
36
+
37
+ def entity_sqname(entity: EntityIdentity) -> str:
38
+ """``cik:0000066740`` for an SEC filer; ``entity:<id>`` under any other scheme."""
39
+ prefix, _ = entity_prefix(entity)
40
+ return f"{prefix}:{entity.cik}"
27
41
 
28
42
 
29
43
  def period_interval(period: Period) -> str:
@@ -28,8 +28,7 @@ import json
28
28
 
29
29
  from ..model import Concept, XbrlModel
30
30
  from ._values import (
31
- CIK_PREFIX,
32
- CIK_SCHEME,
31
+ entity_prefix,
33
32
  entity_sqname,
34
33
  language_tag,
35
34
  period_interval,
@@ -45,7 +44,7 @@ OIM_RESERVED_NAMESPACES: dict[str, str] = {
45
44
  "utr": "http://www.xbrl.org/2009/utr",
46
45
  }
47
46
 
48
- # The SEC entity scheme (CIK_PREFIX / CIK_SCHEME), the period literal and the
47
+ # The entity SQName and its scheme binding, the period literal and the
49
48
  # language form are shared with the Tavi projection — see ``_values``.
50
49
 
51
50
  # Unit measures that mean "no unit" and are therefore left off the fact.
@@ -75,7 +74,8 @@ def to_oim_document(
75
74
  def _namespaces(model: XbrlModel) -> dict[str, str]:
76
75
  """Prefix map: the reserved set, the entity scheme, and the filing's own."""
77
76
  namespaces = dict(OIM_RESERVED_NAMESPACES)
78
- namespaces[CIK_PREFIX] = CIK_SCHEME
77
+ prefix, scheme = entity_prefix(model.entity)
78
+ namespaces[prefix] = scheme
79
79
 
80
80
  by_uri = {uri: prefix for prefix, uri in namespaces.items()}
81
81
  for concept in model.concepts.values():
@@ -95,7 +95,7 @@ def _facts(model: XbrlModel, namespaces: dict[str, str]) -> dict[str, object]:
95
95
  prefixes = {uri: prefix for prefix, uri in namespaces.items()}
96
96
  periods = {period.id: period for period in model.periods}
97
97
  units = {unit.id: unit for unit in model.units}
98
- entity = entity_sqname(model.entity.cik)
98
+ entity = entity_sqname(model.entity)
99
99
  facts: dict[str, object] = {}
100
100
  concepts = model.concepts
101
101
 
@@ -51,8 +51,7 @@ from dataclasses import dataclass, field
51
51
  from ..model import Arc, Concept, Network, Period, XbrlFact, XbrlModel
52
52
  from ..namespaces import TAVI_REPORT_BASE
53
53
  from ._values import (
54
- CIK_PREFIX,
55
- CIK_SCHEME,
54
+ entity_prefix,
56
55
  entity_sqname,
57
56
  language_tag,
58
57
  period_interval,
@@ -206,6 +205,7 @@ LABEL_ROLE_TYPES: dict[str, str] = {
206
205
  "http://xbrl.us/us-gaap/role/label/negatedTotal": "xbrl:negatedTotal",
207
206
  }
208
207
  DEFAULT_LABEL_TYPE = "xbrl:label"
208
+ DOCUMENTATION_LABEL_TYPE = "xbrl:documentation"
209
209
 
210
210
  PRESENTATION_RELATIONSHIP = "xbrl:parent-child"
211
211
  CALCULATION_RELATIONSHIP = "xbrl:summation-item"
@@ -401,16 +401,23 @@ class GapReport:
401
401
  }
402
402
 
403
403
 
404
- def to_tavi(model: XbrlModel, *, report_id: str | None = None) -> str:
404
+ def to_tavi(
405
+ model: XbrlModel, *, report_id: str | None = None, description: str | None = None
406
+ ) -> str:
405
407
  """Project ``model`` into a Tavi compiled-model JSON string."""
406
- document, _ = to_tavi_report(model, report_id=report_id)
408
+ document, _ = to_tavi_report(model, report_id=report_id, description=description)
407
409
  return json.dumps(document, indent=2, sort_keys=False, default=str)
408
410
 
409
411
 
410
412
  def to_tavi_report(
411
- model: XbrlModel, *, report_id: str | None = None
413
+ model: XbrlModel, *, report_id: str | None = None, description: str | None = None
412
414
  ) -> tuple[dict[str, object], GapReport]:
413
- """Project ``model``, returning the document and what it could not express."""
415
+ """Project ``model``, returning the document and what it could not express.
416
+
417
+ ``report_id`` scopes the report's own namespace (the accession by default);
418
+ ``description`` replaces the ``documentInfo`` sentence, which otherwise
419
+ reads the model as an EDGAR filing.
420
+ """
414
421
  report_id = report_id or model.filing.accession
415
422
  gaps = GapReport()
416
423
  namespaces = _namespaces(model, report_id)
@@ -439,7 +446,9 @@ def to_tavi_report(
439
446
  "domainNetworks": dimensional.domain_networks,
440
447
  "members": dimensional.members,
441
448
  "cubes": dimensional.cubes,
442
- "labels": _labels(model, gaps, default_language) + group_labels,
449
+ "labels": _labels(model, gaps, default_language)
450
+ + _entity_labels(model, default_language)
451
+ + group_labels,
443
452
  "networks": networks,
444
453
  "groups": groups,
445
454
  "groupContents": group_contents,
@@ -450,7 +459,8 @@ def to_tavi_report(
450
459
  "documentInfo": {
451
460
  "documentType": DOCTYPE_COMPILED,
452
461
  "namespaces": namespaces,
453
- "description": (
462
+ "description": description
463
+ or (
454
464
  f"{model.filing.form or 'filing'} {model.filing.accession} "
455
465
  f"(CIK {model.filing.cik}) projected from XBRL by xbrlkit"
456
466
  ),
@@ -703,8 +713,10 @@ def _namespaces(model: XbrlModel, report_id: str) -> dict[str, str]:
703
713
  """
704
714
  namespaces = dict(RESERVED_NAMESPACES)
705
715
  namespaces[REPORT_PREFIX] = _report_namespace(report_id)
706
- # The entity scheme: the SQName ``cik:0000066740`` needs it bound.
707
- namespaces[CIK_PREFIX] = CIK_SCHEME
716
+ # The entity scheme: the SQName ``cik:0000066740`` (or ``entity:<id>`` under
717
+ # any other scheme) needs it bound.
718
+ prefix, scheme = entity_prefix(model.entity)
719
+ namespaces[prefix] = scheme
708
720
 
709
721
  by_uri = {uri: prefix for prefix, uri in namespaces.items()}
710
722
  for concept in model.concepts.values():
@@ -761,13 +773,33 @@ def _default_language(model: XbrlModel) -> str:
761
773
  def _entities(model: XbrlModel) -> list[dict[str, object]]:
762
774
  """The reporting entity (section 8.1).
763
775
 
764
- An entity's SQName "includes the scheme and the identifier", so the SEC
765
- scheme is the namespace and the CIK the local name — ``cik:0000066740``, the
766
- same name the OIM projection writes and Arelle's converter emits. It was
767
- previously minted under this report's own namespace, which dropped the
768
- scheme and gave two converters of one filing two different entities.
776
+ An entity's SQName "includes the scheme and the identifier", so the scheme
777
+ is the namespace and the identifier the local name — ``cik:0000066740`` for
778
+ an SEC filer, the same name the OIM projection writes and Arelle's converter
779
+ emits, and ``entity:<id>`` under any other scheme. It was previously minted
780
+ under this report's own namespace, which dropped the scheme and gave two
781
+ converters of one filing two different entities.
769
782
  """
770
- return [{"name": entity_sqname(model.entity.cik)}]
783
+ return [{"name": entity_sqname(model.entity)}]
784
+
785
+
786
+ def _entity_labels(model: XbrlModel, default_language: str) -> list[dict[str, object]]:
787
+ """The entity's name as a label object pointing at the entity (section 5.14).
788
+
789
+ ``forObject`` is "any", and a reader holding the compiled model and nothing
790
+ else — no ``dei`` facts, no EDGAR header — still needs a name to put on the
791
+ report. The entity object itself carries only its SQName.
792
+ """
793
+ if not model.entity.name:
794
+ return []
795
+ return [
796
+ {
797
+ "forObject": entity_sqname(model.entity),
798
+ "labelType": DEFAULT_LABEL_TYPE,
799
+ "value": model.entity.name,
800
+ "language": default_language,
801
+ }
802
+ ]
771
803
 
772
804
 
773
805
  def _units(model: XbrlModel) -> list[dict[str, object]]:
@@ -995,9 +1027,12 @@ def _networks_and_groups(
995
1027
  group_labels: list[dict[str, object]] = []
996
1028
  group_names: dict[str, str] = {}
997
1029
  definitions: dict[str, str] = {}
1030
+ documentations: dict[str, str] = {}
998
1031
  for network in model.networks:
999
1032
  if network.definition and network.role_uri not in definitions:
1000
1033
  definitions[network.role_uri] = network.definition
1034
+ if network.documentation and network.role_uri not in documentations:
1035
+ documentations[network.role_uri] = network.documentation
1001
1036
 
1002
1037
  def group_for(role_uri: str) -> str:
1003
1038
  group_name = group_names.get(role_uri)
@@ -1019,6 +1054,16 @@ def _networks_and_groups(
1019
1054
  "language": default_language,
1020
1055
  }
1021
1056
  )
1057
+ documentation = documentations.get(role_uri)
1058
+ if documentation:
1059
+ group_labels.append(
1060
+ {
1061
+ "forObject": group_name,
1062
+ "labelType": DOCUMENTATION_LABEL_TYPE,
1063
+ "value": documentation,
1064
+ "language": default_language,
1065
+ }
1066
+ )
1022
1067
  return group_name
1023
1068
 
1024
1069
  for index, network in enumerate(model.networks):
@@ -1089,7 +1134,7 @@ def _facts(
1089
1134
  """
1090
1135
  periods = {period.id: period for period in model.periods}
1091
1136
  units = {unit.id: unit for unit in model.units}
1092
- entity = entity_sqname(model.entity.cik)
1137
+ entity = entity_sqname(model.entity)
1093
1138
  facts: list[dict[str, object]] = []
1094
1139
 
1095
1140
  for index, fact in enumerate(model.facts):
@@ -26,6 +26,12 @@ Continuations nest: a table text block inside a revenue note continues in a
26
26
  ``ix:continuation`` that sits physically inside the note's own continuation.
27
27
  Every continuation is registered with its offsets, nested or not, and a chain
28
28
  appends a link only when no appended link already contains it.
29
+
30
+ A concept tagged more than once — a purchase-price table per acquisition, a
31
+ roll-forward per period — is one section holding every distinct occurrence
32
+ in document order; keeping the first alone lost the second table. Content
33
+ inside ``ix:exclude`` (the page header some filers place inside a tagged
34
+ block) is not part of the fact and is dropped, as Arelle drops it.
29
35
  """
30
36
 
31
37
  from __future__ import annotations
@@ -152,6 +158,12 @@ class _Block:
152
158
 
153
159
  _TAG_NAME_END = " \t\r\n/>"
154
160
 
161
+ # Content a filer marks as not part of the fact: page headers and footers
162
+ # inside a tagged block. Applied to one section's HTML, never the document.
163
+ _EXCLUDE_RE = re.compile(
164
+ r"<ix:exclude\b[^>]*>.*?</ix:exclude\s*>", re.IGNORECASE | re.DOTALL
165
+ )
166
+
155
167
 
156
168
  def _scan_blocks(html: str, tag: str) -> list[_Block]:
157
169
  """Every ``<tag>`` block in document order, nested blocks included.
@@ -222,14 +234,21 @@ class iXBRLParser: # noqa: N801 — the established name across consumers
222
234
  ``ix:continuation`` chain resolved, its numeric facts listed — nested
223
235
  ones included: a policy block inside an accounting-policies note is its
224
236
  own section as well as part of the note's text, because its continued
225
- tail belongs to its own chain and to no other. A section over
226
- ``part_size`` is returned as consecutive parts that share the
227
- ``section_id`` and carry ``part``/``part_count``.
237
+ tail belongs to its own chain and to no other. A concept tagged more
238
+ than once is one section holding every distinct occurrence in document
239
+ order (the same table tagged twice with different markup counts once);
240
+ ``ix:exclude`` content is dropped. A section over ``part_size`` is
241
+ returned as consecutive parts that share the ``section_id`` and carry
242
+ ``part``/``part_count``.
228
243
  """
229
244
  continuations = self._build_continuation_map(html)
230
245
 
231
- sections: list[iXBRLSection] = []
232
- seen_ids: set[str] = set()
246
+ # Per element name, in document order: each distinct occurrence's HTML
247
+ # and text, the blocks already taken, and the texts already seen.
248
+ htmls: dict[str, list[str]] = {}
249
+ texts: dict[str, list[str]] = {}
250
+ taken: dict[str, list[_Block]] = {}
251
+ seen: dict[str, set[str]] = {}
233
252
 
234
253
  for block in _scan_blocks(html, "ix:nonNumeric"):
235
254
  name_match = re.search(
@@ -243,10 +262,9 @@ class iXBRLParser: # noqa: N801 — the established name across consumers
243
262
  if element_name.startswith(("dei:", "ecd:")):
244
263
  continue
245
264
 
246
- # Deduplicate (the same element can appear more than once)
247
- if element_name in seen_ids:
265
+ # A block nested in an earlier block of the same name is already in it
266
+ if any(outer.contains(block) for outer in taken.get(element_name, ())):
248
267
  continue
249
- seen_ids.add(element_name)
250
268
 
251
269
  full_html = block.content(html)
252
270
  continued_at = _continued_at(block.attrs)
@@ -254,14 +272,25 @@ class iXBRLParser: # noqa: N801 — the established name across consumers
254
272
  full_html += self._resolve_continuation_chain(
255
273
  continued_at, continuations, html, covered=[block]
256
274
  )
257
-
258
- xbrl_elements = _extract_elements_from_block(full_html)
275
+ full_html = _EXCLUDE_RE.sub(" ", full_html)
259
276
  content = _strip_html(full_html)
260
277
 
278
+ key = " ".join(content.split())
279
+ if key in seen.setdefault(element_name, set()):
280
+ continue
281
+ seen[element_name].add(key)
282
+ taken.setdefault(element_name, []).append(block)
283
+ htmls.setdefault(element_name, []).append(full_html)
284
+ texts.setdefault(element_name, []).append(content)
285
+
286
+ sections: list[iXBRLSection] = []
287
+ for element_name, occurrences in texts.items():
288
+ content = "\n\n".join(occurrences)
261
289
  word_count = len(content.split())
262
290
  if word_count < MIN_SECTION_WORDS:
263
291
  continue
264
292
 
293
+ xbrl_elements = _extract_elements_from_block("".join(htmls[element_name]))
265
294
  section_label = _label_from_element_name(element_name)
266
295
  parts = split_text(content, self.part_size)
267
296
  for index, part_text in enumerate(parts, start=1):
@@ -281,7 +310,7 @@ class iXBRLParser: # noqa: N801 — the established name across consumers
281
310
  logger.debug(
282
311
  "iXBRL parsed: %d disclosure documents (%d sections), %d elements",
283
312
  len(sections),
284
- len(seen_ids),
313
+ len(texts),
285
314
  sum(s.element_count for s in sections if s.part == 1),
286
315
  )
287
316
  return sections
@@ -1,24 +0,0 @@
1
- """xbrlkit — a SEC XBRL filing to a portable ``holon.jsonld``.
2
-
3
- Fetch a filing from EDGAR, parse it into the neutral :class:`XbrlModel`, and
4
- project that into the canonical scene/boundary/projection holon that renders
5
- offline in the RoboSystems holon viewer.
6
- """
7
-
8
- from __future__ import annotations
9
-
10
- from importlib.metadata import PackageNotFoundError, version
11
-
12
- from .model import XbrlModel
13
-
14
-
15
- def _get_version() -> str:
16
- try:
17
- return version("xbrlkit")
18
- except PackageNotFoundError:
19
- return "0.0.0+development"
20
-
21
-
22
- __version__ = _get_version()
23
-
24
- __all__ = ("XbrlModel", "__version__")
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes