xbrlkit 0.4.0__tar.gz → 0.4.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/PKG-INFO +65 -7
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/README.md +64 -6
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/pyproject.toml +1 -1
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/tests/test_oim.py +17 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/tests/test_tavi.py +114 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/tests/test_text_ixbrl.py +77 -5
- xbrlkit-0.4.2/xbrlkit/__init__.py +28 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/model.py +4 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/serialize/_values.py +21 -7
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/serialize/oim.py +5 -5
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/serialize/tavi.py +62 -17
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/text/ixbrl.py +40 -11
- xbrlkit-0.4.0/xbrlkit/__init__.py +0 -24
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/.gitignore +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/LICENSE +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/pytest.ini +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/ruff.toml +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/tests/__init__.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/tests/conftest.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/tests/test_arelle_cache.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/tests/test_config.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/tests/test_edgar.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/tests/test_efts.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/tests/test_lpg.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/tests/test_parity.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/tests/test_parse.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/tests/test_query.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/tests/test_serialize.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/tests/test_text_narrative.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/tests/test_text_parts.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/tests/test_text_tables.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/__init__.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/__init__.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/conf/README.md +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/conf/extractTestcase.sh +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/conf/extractTestcase.xsl +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/conf/runIxtSecTests.sh +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/conf/saxon9.jar +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/conf/testcase.xml +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/conf/tests.xml +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/conf/tests.xsd +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/text2num.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/transformationRegistry/registry/ixt-sec-boolballotbox.xml +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/transformationRegistry/registry/ixt-sec-countrynameen.xml +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/transformationRegistry/registry/ixt-sec-datequarterend.xml +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/transformationRegistry/registry/ixt-sec-durday.xml +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/transformationRegistry/registry/ixt-sec-durhour.xml +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/transformationRegistry/registry/ixt-sec-durmonth.xml +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/transformationRegistry/registry/ixt-sec-durweek.xml +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/transformationRegistry/registry/ixt-sec-durwordsen.xml +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/transformationRegistry/registry/ixt-sec-duryear.xml +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/transformationRegistry/registry/ixt-sec-edgarprovcountryen.xml +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/transformationRegistry/registry/ixt-sec-entityfilercategoryen.xml +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/transformationRegistry/registry/ixt-sec-exchnameen.xml +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/transformationRegistry/registry/ixt-sec-numwordsen.xml +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/transformationRegistry/registry/ixt-sec-stateprovnameen.xml +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/transformationRegistry/registry/ixt-sec-yesnoballotbox.xml +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/transformationRegistry/registry/transform-registry.xml +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/transformationRegistry/schema/inlinexbrl-sec-transformation.xsd +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/ontology/v1/context.jsonld +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/ontology/v1/shapes.ttl +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/cli.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/config.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/edgar/__init__.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/edgar/client.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/edgar/download.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/edgar/efts.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/edgar/rate_limit.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/namespaces.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/parse/__init__.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/parse/arelle_load.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/parse/cache.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/parse/ids.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/parse/to_model.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/py.typed +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/query.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/schema/__init__.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/serialize/__init__.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/serialize/_kernel/__init__.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/serialize/_kernel/bundle.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/serialize/_kernel/context.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/serialize/_kernel/holon.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/serialize/_kernel/jsonld.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/serialize/classify.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/serialize/graph.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/serialize/holon.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/serialize/lpg.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/text/__init__.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/text/narrative.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/text/parts.py +0 -0
- {xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/text/tables.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: xbrlkit
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.2
|
|
4
4
|
Summary: Work with XBRL filings above Arelle: one parse, portable models
|
|
5
5
|
Project-URL: Repository, https://github.com/RoboFinSystems/xbrlkit
|
|
6
6
|
Project-URL: Homepage, https://github.com/RoboFinSystems/xbrlkit#readme
|
|
@@ -60,12 +60,16 @@ it **once** into a neutral typed model, and project that model into whichever
|
|
|
60
60
|
portable representation you need.
|
|
61
61
|
|
|
62
62
|
```
|
|
63
|
-
EDGAR ──▶ Arelle ──▶ XbrlModel ──┬──▶ holon.jsonld
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
63
|
+
EDGAR ──▶ Arelle ──▶ XbrlModel ──┬──▶ holon.jsonld (RDF / JSON-LD)
|
|
64
|
+
│ ├──▶ Tavi (compiled model)
|
|
65
|
+
│ ├──▶ xBRL-JSON (OIM)
|
|
66
|
+
│ └──▶ property graph (parquet, .lbug)
|
|
67
|
+
└──▶ primary HTML ──▶ xbrlkit.text ──▶ sections (text blocks, Items, tables)
|
|
67
68
|
```
|
|
68
69
|
|
|
70
|
+
Four projections hang off the model. A fifth surface, the filing's text, reads the
|
|
71
|
+
primary HTML document directly and needs neither Arelle nor the network.
|
|
72
|
+
|
|
69
73
|
Arelle stays the parser — nobody should reimplement DTS resolution. What it does
|
|
70
74
|
not give you is anything ergonomic to *hold*: `ModelXbrl` is a large mutable
|
|
71
75
|
object graph tied to a controller you have to close. `XbrlModel` is the answer to
|
|
@@ -141,7 +145,7 @@ network — and returns its text as sections:
|
|
|
141
145
|
|
|
142
146
|
| Parser | Sections | Notes |
|
|
143
147
|
| --- | --- | --- |
|
|
144
|
-
| `iXBRLParser` | every inline-XBRL text block (notes, policies, tables), with the XBRL element names it contains | `ix:continuation` chains resolved; nested continuations and nested text blocks included |
|
|
148
|
+
| `iXBRLParser` | every inline-XBRL text block (notes, policies, tables), with the XBRL element names it contains | `ix:continuation` chains resolved; nested continuations and nested text blocks included; a concept tagged more than once is one section holding every occurrence; `ix:exclude` page furniture dropped |
|
|
145
149
|
| `NarrativeExtractor` | the 10-K / 10-Q Items — Business, Risk Factors, Cybersecurity, Properties, MD&A, Market Risk | table-of-contents rows and cross-references rejected; a 10-Q's Part I and Part II Items kept apart |
|
|
146
150
|
|
|
147
151
|
Both render HTML tables as markdown pipe tables and split a long section into
|
|
@@ -207,7 +211,14 @@ SEC_GOV_USER_AGENT="Your Name your@email.com"
|
|
|
207
211
|
# Build a holon.jsonld from a specific filing (-> ./output/)
|
|
208
212
|
xbrlkit build --cik 320193 --accno 0000320193-23-000106
|
|
209
213
|
|
|
210
|
-
#
|
|
214
|
+
# The other projections: Tavi (plus its .tavi.gaps.json sidecar), xBRL-JSON,
|
|
215
|
+
# the property graph (needs the lpg extra), or every one of them
|
|
216
|
+
xbrlkit build --cik 320193 --accno 0000320193-23-000106 --format tavi
|
|
217
|
+
xbrlkit build --cik 320193 --accno 0000320193-23-000106 --format oim
|
|
218
|
+
xbrlkit build --cik 320193 --accno 0000320193-23-000106 --format lpg
|
|
219
|
+
xbrlkit build --cik 320193 --accno 0000320193-23-000106 --format all
|
|
220
|
+
|
|
221
|
+
# Fetch the latest filing for a ticker (-> ./output/); --form and --n filter
|
|
211
222
|
xbrlkit fetch --ticker NVDA
|
|
212
223
|
|
|
213
224
|
# Query consolidated facts in a built holon (in-memory SPARQL)
|
|
@@ -217,6 +228,31 @@ xbrlkit query --in output/0000320193-23-000106.holon.jsonld --element us-gaap:As
|
|
|
217
228
|
From a source checkout, `just` wraps the same CLI as a shorthand:
|
|
218
229
|
`just build 320193 0000320193-23-000106` and `just fetch NVDA`.
|
|
219
230
|
|
|
231
|
+
## EDGAR
|
|
232
|
+
|
|
233
|
+
`xbrlkit.edgar` is the fetch layer the CLI uses, exposed for hosts that discover
|
|
234
|
+
and download filings themselves: synchronous `requests`, local-filesystem output,
|
|
235
|
+
and EDGAR's two throttle signatures — a 429, and an empty 200 — ridden out with
|
|
236
|
+
a bounded wait-and-retry (`EdgarThrottled` when the budget is spent).
|
|
237
|
+
|
|
238
|
+
| | |
|
|
239
|
+
| --- | --- |
|
|
240
|
+
| `EdgarClient` | ticker → CIK, a company's filing list (`list_filings`, by form), `company_info`, one filing by accession (`get_filing_ref`) |
|
|
241
|
+
| `EftsClient` / `query_efts` | bulk discovery through EDGAR full-text search: by form, year or quarter, across every filer |
|
|
242
|
+
| `download_filing` / `fetch` | the XBRL zip for one accession, unpacked to a directory |
|
|
243
|
+
|
|
244
|
+
```python
|
|
245
|
+
from pathlib import Path
|
|
246
|
+
from xbrlkit.edgar import EdgarClient, download_filing
|
|
247
|
+
|
|
248
|
+
client = EdgarClient()
|
|
249
|
+
cik = client.ticker_to_cik("MMM")
|
|
250
|
+
latest = client.list_filings(cik, forms=["10-K"])[0]
|
|
251
|
+
package = download_filing(client, cik, latest.accession, Path("data")) # the Arelle load target
|
|
252
|
+
```
|
|
253
|
+
|
|
254
|
+
The SEC User-Agent is required here as everywhere (see [SEC User-Agent](#sec-user-agent)).
|
|
255
|
+
|
|
220
256
|
## Arelle cache
|
|
221
257
|
|
|
222
258
|
Arelle resolves a filing's DTS by fetching every schema and linkbase it imports —
|
|
@@ -242,6 +278,28 @@ xbrlkit cache extract --bundle schemas.tar.gz # seed a container's cache at buil
|
|
|
242
278
|
network; a miss is then an error, not a fetch. A host that builds its own Arelle
|
|
243
279
|
controller gets the same policy from `xbrlkit.parse.configure_webcache(cntlr, cache_dir)`.
|
|
244
280
|
|
|
281
|
+
## Where it runs
|
|
282
|
+
|
|
283
|
+
**RoboSystems.** The platform's SEC pipeline is built on this package: filings are
|
|
284
|
+
parsed with `xbrlkit.parse` (the platform's own Arelle controller, with
|
|
285
|
+
`register_sec_transforms` and the cache policy from `configure_webcache`), projected
|
|
286
|
+
with `to_holon`, `to_tavi_report` and the property-graph tables, the shared `sec`
|
|
287
|
+
graph is declared from `xbrlkit.schema`, and the full-text index behind its document
|
|
288
|
+
search is built from `xbrlkit.text`.
|
|
289
|
+
|
|
290
|
+
**Filing Ladder.** The [Filing Ladder](https://github.com/HarbingerFinLab/filing-ladder)
|
|
291
|
+
benchmark — one filing handed to the same language model in every representation —
|
|
292
|
+
built its 26-filing corpus of 2024–2025 10-Ks and 10-Qs with this package: the Tavi
|
|
293
|
+
compiled model, the holon, the per-filing property graph, and both text parsers. Each
|
|
294
|
+
projection is a rung of the ladder, so the
|
|
295
|
+
[v0.1 results](https://github.com/HarbingerFinLab/filing-ladder/blob/main/results/v0.1-sonnet-5/README.md)
|
|
296
|
+
are also a measurement of what a model can do with each of these outputs. Before that
|
|
297
|
+
run, every text-block section the parsers produce was checked against the filing's own
|
|
298
|
+
text-block facts as Arelle resolves them, on all 26 filings, and the property-graph
|
|
299
|
+
projection was checked row for row against the platform's processor. The two defects
|
|
300
|
+
those checks found in the text layer were fixed in 0.4.1 and are disclosed in the
|
|
301
|
+
benchmark's protocol.
|
|
302
|
+
|
|
245
303
|
## View & explore
|
|
246
304
|
|
|
247
305
|
Built holons render in the **RoboSystems Holon Viewer** — a browser-based reader
|
|
@@ -7,12 +7,16 @@ it **once** into a neutral typed model, and project that model into whichever
|
|
|
7
7
|
portable representation you need.
|
|
8
8
|
|
|
9
9
|
```
|
|
10
|
-
EDGAR ──▶ Arelle ──▶ XbrlModel ──┬──▶ holon.jsonld
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
10
|
+
EDGAR ──▶ Arelle ──▶ XbrlModel ──┬──▶ holon.jsonld (RDF / JSON-LD)
|
|
11
|
+
│ ├──▶ Tavi (compiled model)
|
|
12
|
+
│ ├──▶ xBRL-JSON (OIM)
|
|
13
|
+
│ └──▶ property graph (parquet, .lbug)
|
|
14
|
+
└──▶ primary HTML ──▶ xbrlkit.text ──▶ sections (text blocks, Items, tables)
|
|
14
15
|
```
|
|
15
16
|
|
|
17
|
+
Four projections hang off the model. A fifth surface, the filing's text, reads the
|
|
18
|
+
primary HTML document directly and needs neither Arelle nor the network.
|
|
19
|
+
|
|
16
20
|
Arelle stays the parser — nobody should reimplement DTS resolution. What it does
|
|
17
21
|
not give you is anything ergonomic to *hold*: `ModelXbrl` is a large mutable
|
|
18
22
|
object graph tied to a controller you have to close. `XbrlModel` is the answer to
|
|
@@ -88,7 +92,7 @@ network — and returns its text as sections:
|
|
|
88
92
|
|
|
89
93
|
| Parser | Sections | Notes |
|
|
90
94
|
| --- | --- | --- |
|
|
91
|
-
| `iXBRLParser` | every inline-XBRL text block (notes, policies, tables), with the XBRL element names it contains | `ix:continuation` chains resolved; nested continuations and nested text blocks included |
|
|
95
|
+
| `iXBRLParser` | every inline-XBRL text block (notes, policies, tables), with the XBRL element names it contains | `ix:continuation` chains resolved; nested continuations and nested text blocks included; a concept tagged more than once is one section holding every occurrence; `ix:exclude` page furniture dropped |
|
|
92
96
|
| `NarrativeExtractor` | the 10-K / 10-Q Items — Business, Risk Factors, Cybersecurity, Properties, MD&A, Market Risk | table-of-contents rows and cross-references rejected; a 10-Q's Part I and Part II Items kept apart |
|
|
93
97
|
|
|
94
98
|
Both render HTML tables as markdown pipe tables and split a long section into
|
|
@@ -154,7 +158,14 @@ SEC_GOV_USER_AGENT="Your Name your@email.com"
|
|
|
154
158
|
# Build a holon.jsonld from a specific filing (-> ./output/)
|
|
155
159
|
xbrlkit build --cik 320193 --accno 0000320193-23-000106
|
|
156
160
|
|
|
157
|
-
#
|
|
161
|
+
# The other projections: Tavi (plus its .tavi.gaps.json sidecar), xBRL-JSON,
|
|
162
|
+
# the property graph (needs the lpg extra), or every one of them
|
|
163
|
+
xbrlkit build --cik 320193 --accno 0000320193-23-000106 --format tavi
|
|
164
|
+
xbrlkit build --cik 320193 --accno 0000320193-23-000106 --format oim
|
|
165
|
+
xbrlkit build --cik 320193 --accno 0000320193-23-000106 --format lpg
|
|
166
|
+
xbrlkit build --cik 320193 --accno 0000320193-23-000106 --format all
|
|
167
|
+
|
|
168
|
+
# Fetch the latest filing for a ticker (-> ./output/); --form and --n filter
|
|
158
169
|
xbrlkit fetch --ticker NVDA
|
|
159
170
|
|
|
160
171
|
# Query consolidated facts in a built holon (in-memory SPARQL)
|
|
@@ -164,6 +175,31 @@ xbrlkit query --in output/0000320193-23-000106.holon.jsonld --element us-gaap:As
|
|
|
164
175
|
From a source checkout, `just` wraps the same CLI as a shorthand:
|
|
165
176
|
`just build 320193 0000320193-23-000106` and `just fetch NVDA`.
|
|
166
177
|
|
|
178
|
+
## EDGAR
|
|
179
|
+
|
|
180
|
+
`xbrlkit.edgar` is the fetch layer the CLI uses, exposed for hosts that discover
|
|
181
|
+
and download filings themselves: synchronous `requests`, local-filesystem output,
|
|
182
|
+
and EDGAR's two throttle signatures — a 429, and an empty 200 — ridden out with
|
|
183
|
+
a bounded wait-and-retry (`EdgarThrottled` when the budget is spent).
|
|
184
|
+
|
|
185
|
+
| | |
|
|
186
|
+
| --- | --- |
|
|
187
|
+
| `EdgarClient` | ticker → CIK, a company's filing list (`list_filings`, by form), `company_info`, one filing by accession (`get_filing_ref`) |
|
|
188
|
+
| `EftsClient` / `query_efts` | bulk discovery through EDGAR full-text search: by form, year or quarter, across every filer |
|
|
189
|
+
| `download_filing` / `fetch` | the XBRL zip for one accession, unpacked to a directory |
|
|
190
|
+
|
|
191
|
+
```python
|
|
192
|
+
from pathlib import Path
|
|
193
|
+
from xbrlkit.edgar import EdgarClient, download_filing
|
|
194
|
+
|
|
195
|
+
client = EdgarClient()
|
|
196
|
+
cik = client.ticker_to_cik("MMM")
|
|
197
|
+
latest = client.list_filings(cik, forms=["10-K"])[0]
|
|
198
|
+
package = download_filing(client, cik, latest.accession, Path("data")) # the Arelle load target
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
The SEC User-Agent is required here as everywhere (see [SEC User-Agent](#sec-user-agent)).
|
|
202
|
+
|
|
167
203
|
## Arelle cache
|
|
168
204
|
|
|
169
205
|
Arelle resolves a filing's DTS by fetching every schema and linkbase it imports —
|
|
@@ -189,6 +225,28 @@ xbrlkit cache extract --bundle schemas.tar.gz # seed a container's cache at buil
|
|
|
189
225
|
network; a miss is then an error, not a fetch. A host that builds its own Arelle
|
|
190
226
|
controller gets the same policy from `xbrlkit.parse.configure_webcache(cntlr, cache_dir)`.
|
|
191
227
|
|
|
228
|
+
## Where it runs
|
|
229
|
+
|
|
230
|
+
**RoboSystems.** The platform's SEC pipeline is built on this package: filings are
|
|
231
|
+
parsed with `xbrlkit.parse` (the platform's own Arelle controller, with
|
|
232
|
+
`register_sec_transforms` and the cache policy from `configure_webcache`), projected
|
|
233
|
+
with `to_holon`, `to_tavi_report` and the property-graph tables, the shared `sec`
|
|
234
|
+
graph is declared from `xbrlkit.schema`, and the full-text index behind its document
|
|
235
|
+
search is built from `xbrlkit.text`.
|
|
236
|
+
|
|
237
|
+
**Filing Ladder.** The [Filing Ladder](https://github.com/HarbingerFinLab/filing-ladder)
|
|
238
|
+
benchmark — one filing handed to the same language model in every representation —
|
|
239
|
+
built its 26-filing corpus of 2024–2025 10-Ks and 10-Qs with this package: the Tavi
|
|
240
|
+
compiled model, the holon, the per-filing property graph, and both text parsers. Each
|
|
241
|
+
projection is a rung of the ladder, so the
|
|
242
|
+
[v0.1 results](https://github.com/HarbingerFinLab/filing-ladder/blob/main/results/v0.1-sonnet-5/README.md)
|
|
243
|
+
are also a measurement of what a model can do with each of these outputs. Before that
|
|
244
|
+
run, every text-block section the parsers produce was checked against the filing's own
|
|
245
|
+
text-block facts as Arelle resolves them, on all 26 filings, and the property-graph
|
|
246
|
+
projection was checked row for row against the platform's processor. The two defects
|
|
247
|
+
those checks found in the text layer were fixed in 0.4.1 and are disclosed in the
|
|
248
|
+
benchmark's protocol.
|
|
249
|
+
|
|
192
250
|
## View & explore
|
|
193
251
|
|
|
194
252
|
Built holons render in the **RoboSystems Holon Viewer** — a browser-based reader
|
|
@@ -153,6 +153,23 @@ def test_document_envelope() -> None:
|
|
|
153
153
|
assert info["namespaces"]["cik"] == "http://www.sec.gov/CIK"
|
|
154
154
|
|
|
155
155
|
|
|
156
|
+
def test_non_sec_entity_binds_its_own_scheme() -> None:
|
|
157
|
+
"""An entity under any scheme but the SEC's is ``entity:<id>``, scheme bound."""
|
|
158
|
+
model = _model().model_copy(
|
|
159
|
+
update={
|
|
160
|
+
"entity": EntityIdentity(cik="ent_01K3ZQ", scheme="http://robosystems.ai/entity")
|
|
161
|
+
}
|
|
162
|
+
)
|
|
163
|
+
document = to_oim_document(model)
|
|
164
|
+
namespaces = document["documentInfo"]["namespaces"]
|
|
165
|
+
assert namespaces["entity"] == "http://robosystems.ai/entity"
|
|
166
|
+
assert "cik" not in namespaces
|
|
167
|
+
assert all(
|
|
168
|
+
fact["dimensions"]["entity"] == "entity:ent_01K3ZQ"
|
|
169
|
+
for fact in document["facts"].values()
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
|
|
156
173
|
def test_instant_period_is_the_exclusive_next_midnight() -> None:
|
|
157
174
|
"""The close of 2024-12-31 is written as the instant after it."""
|
|
158
175
|
assert _by_concept("us-gaap:Assets")["dimensions"]["period"] == "2025-01-01T00:00:00"
|
|
@@ -407,6 +407,85 @@ def test_entity_sqname_keeps_the_scheme() -> None:
|
|
|
407
407
|
assert _document()["xbrlModel"]["entities"] == [{"name": "cik:0001234567"}]
|
|
408
408
|
|
|
409
409
|
|
|
410
|
+
def _ledger_model() -> XbrlModel:
|
|
411
|
+
"""The same model re-homed on an entity that is not an EDGAR filer."""
|
|
412
|
+
return _model().model_copy(
|
|
413
|
+
update={
|
|
414
|
+
"entity": EntityIdentity(
|
|
415
|
+
cik="ent_01K3ZQ", scheme="http://robosystems.ai/entity", name="Acme LLC"
|
|
416
|
+
)
|
|
417
|
+
}
|
|
418
|
+
)
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
def test_non_sec_entity_binds_its_own_scheme_under_entity() -> None:
|
|
422
|
+
"""Section 8.1 again: the prefix follows the scheme, not the SEC.
|
|
423
|
+
|
|
424
|
+
A model built from a ledger's own report has no CIK. Its entity is still
|
|
425
|
+
scheme + identifier — written ``entity:<id>`` under the scheme it declares,
|
|
426
|
+
with no ``cik`` binding at all.
|
|
427
|
+
"""
|
|
428
|
+
document, _ = to_tavi_report(_ledger_model())
|
|
429
|
+
namespaces = document["documentInfo"]["namespaces"]
|
|
430
|
+
assert namespaces["entity"] == "http://robosystems.ai/entity"
|
|
431
|
+
assert "cik" not in namespaces
|
|
432
|
+
assert document["xbrlModel"]["entities"] == [{"name": "entity:ent_01K3ZQ"}]
|
|
433
|
+
assert all(
|
|
434
|
+
fact["factDimensions"]["xbrl:entity"] == "entity:ent_01K3ZQ"
|
|
435
|
+
for fact in document["xbrlModel"]["facts"]
|
|
436
|
+
)
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
def test_entity_name_is_a_label_on_the_entity() -> None:
|
|
440
|
+
"""The entity's name is a label object pointing at the entity (5.14).
|
|
441
|
+
|
|
442
|
+
A reader holding the compiled model and nothing else — no ``dei`` facts, no
|
|
443
|
+
EDGAR header — still needs a name to put on the report.
|
|
444
|
+
"""
|
|
445
|
+
labels = [
|
|
446
|
+
entry
|
|
447
|
+
for entry in _document()["xbrlModel"]["labels"]
|
|
448
|
+
if entry["forObject"] == "cik:0001234567"
|
|
449
|
+
]
|
|
450
|
+
assert labels == [
|
|
451
|
+
{
|
|
452
|
+
"forObject": "cik:0001234567",
|
|
453
|
+
"labelType": "xbrl:label",
|
|
454
|
+
"value": "Acme Corp",
|
|
455
|
+
"language": "en-US",
|
|
456
|
+
}
|
|
457
|
+
]
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
def test_entity_without_a_name_has_no_label() -> None:
|
|
461
|
+
model = _model().model_copy(update={"entity": EntityIdentity(cik="0001234567")})
|
|
462
|
+
document, _ = to_tavi_report(model)
|
|
463
|
+
assert not [
|
|
464
|
+
entry
|
|
465
|
+
for entry in document["xbrlModel"]["labels"]
|
|
466
|
+
if entry["forObject"] == "cik:0001234567"
|
|
467
|
+
]
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
def test_description_reads_the_filing_unless_one_is_supplied() -> None:
|
|
471
|
+
assert (
|
|
472
|
+
_document()["documentInfo"]["description"]
|
|
473
|
+
== "10-K 0000000000-24-000001 (CIK 0001234567) projected from XBRL by xbrlkit"
|
|
474
|
+
)
|
|
475
|
+
document, _ = to_tavi_report(
|
|
476
|
+
_ledger_model(), description="RoboLedger report r1 g2 (Acme LLC)"
|
|
477
|
+
)
|
|
478
|
+
assert document["documentInfo"]["description"] == (
|
|
479
|
+
"RoboLedger report r1 g2 (Acme LLC)"
|
|
480
|
+
)
|
|
481
|
+
assert (
|
|
482
|
+
json.loads(to_tavi(_ledger_model(), description="custom"))["documentInfo"][
|
|
483
|
+
"description"
|
|
484
|
+
]
|
|
485
|
+
== "custom"
|
|
486
|
+
)
|
|
487
|
+
|
|
488
|
+
|
|
410
489
|
def test_calculation_relationship_carries_weight_only() -> None:
|
|
411
490
|
"""Section 14.3.1: weight is required; reconciliation is a flag, not asserted."""
|
|
412
491
|
networks = _document()["xbrlModel"]["networks"]
|
|
@@ -463,6 +542,41 @@ def test_group_definition_is_a_label_with_the_filing_language() -> None:
|
|
|
463
542
|
assert all("language" in entry for entry in model["labels"])
|
|
464
543
|
|
|
465
544
|
|
|
545
|
+
def test_network_documentation_is_a_second_label_on_the_group() -> None:
|
|
546
|
+
"""A role's longer reading rides as an ``xbrl:documentation`` label.
|
|
547
|
+
|
|
548
|
+
A producer whose definition is a composed sort key keeps the role's own
|
|
549
|
+
name in ``documentation``; a network without one adds no label.
|
|
550
|
+
"""
|
|
551
|
+
model = _model()
|
|
552
|
+
networks = [
|
|
553
|
+
n.model_copy(update={"documentation": "Consolidated Balance Sheets"})
|
|
554
|
+
if n.role_uri == "http://example.com/role/BalanceSheet"
|
|
555
|
+
else n
|
|
556
|
+
for n in model.networks
|
|
557
|
+
]
|
|
558
|
+
document, _ = to_tavi_report(model.model_copy(update={"networks": networks}))
|
|
559
|
+
xbrl_model = document["xbrlModel"]
|
|
560
|
+
group_name = next(
|
|
561
|
+
g["name"]
|
|
562
|
+
for g in xbrl_model["groups"]
|
|
563
|
+
if g["groupURI"] == "http://example.com/role/BalanceSheet"
|
|
564
|
+
)
|
|
565
|
+
labels = {
|
|
566
|
+
entry["labelType"]: entry["value"]
|
|
567
|
+
for entry in xbrl_model["labels"]
|
|
568
|
+
if entry["forObject"] == group_name
|
|
569
|
+
}
|
|
570
|
+
assert labels == {
|
|
571
|
+
"xbrl:label": "Balance Sheet",
|
|
572
|
+
"xbrl:documentation": "Consolidated Balance Sheets",
|
|
573
|
+
}
|
|
574
|
+
plain = _document()["xbrlModel"]
|
|
575
|
+
assert not [
|
|
576
|
+
entry for entry in plain["labels"] if entry["labelType"] == "xbrl:documentation"
|
|
577
|
+
]
|
|
578
|
+
|
|
579
|
+
|
|
466
580
|
def test_shares_map_to_the_accounting_module() -> None:
|
|
467
581
|
"""No core datatype types a share count; the xbrla module does."""
|
|
468
582
|
document, gaps = to_tavi_report(_model())
|
|
@@ -5,7 +5,7 @@ Covers:
|
|
|
5
5
|
- Continuation chain resolution, nested continuations included
|
|
6
6
|
- Element extraction from nested ix:nonFraction tags
|
|
7
7
|
- Label derivation from element qnames
|
|
8
|
-
- Filtering (DEI/ECD skip, min word count,
|
|
8
|
+
- Filtering (DEI/ECD skip, min word count), repeated tags, ix:exclude
|
|
9
9
|
- Splitting a long section into parts
|
|
10
10
|
"""
|
|
11
11
|
|
|
@@ -248,20 +248,92 @@ class TestiXBRLParser:
|
|
|
248
248
|
"""
|
|
249
249
|
assert iXBRLParser().parse(html) == []
|
|
250
250
|
|
|
251
|
-
def
|
|
251
|
+
def test_same_text_tagged_twice_counts_once(self):
|
|
252
|
+
"""The same table tagged twice with different markup (Smucker's
|
|
253
|
+
intangibles-acquired table) is one occurrence, not two."""
|
|
252
254
|
html = f"""
|
|
253
255
|
<html><body>
|
|
254
256
|
<ix:nonNumeric contextRef="c-1" name="us-gaap:RevenueTextBlock" id="f-1">
|
|
255
|
-
|
|
257
|
+
<p>Same occurrence {_FILLER}</p>
|
|
256
258
|
</ix:nonNumeric>
|
|
257
259
|
<ix:nonNumeric contextRef="c-2" name="us-gaap:RevenueTextBlock" id="f-2">
|
|
258
|
-
|
|
260
|
+
<div><span>Same occurrence</span> {_FILLER}</div>
|
|
259
261
|
</ix:nonNumeric>
|
|
260
262
|
</body></html>
|
|
261
263
|
"""
|
|
262
264
|
sections = iXBRLParser().parse(html)
|
|
263
265
|
assert len(sections) == 1
|
|
264
|
-
assert "
|
|
266
|
+
assert sections[0].content.count("Same occurrence") == 1
|
|
267
|
+
|
|
268
|
+
def test_merges_distinct_occurrences_of_a_concept(self):
|
|
269
|
+
"""A concept tagged more than once — Microsoft's purchase-price table for
|
|
270
|
+
each of two acquisitions, loanDepot's servicing-assets roll-forward per
|
|
271
|
+
period — is one section holding every occurrence in document order.
|
|
272
|
+
Keeping the first alone lost the second table."""
|
|
273
|
+
html = f"""
|
|
274
|
+
<html><body>
|
|
275
|
+
<ix:nonNumeric contextRef="c-1" name="us-gaap:ScheduleOfBusinessAcquisitionsByAcquisitionTextBlock" id="f-1">
|
|
276
|
+
<p>Nuance purchase price {_FILLER}
|
|
277
|
+
<ix:nonFraction name="us-gaap:Goodwill" contextRef="c-1">16326</ix:nonFraction></p>
|
|
278
|
+
</ix:nonNumeric>
|
|
279
|
+
<p>Other text between the two tables.</p>
|
|
280
|
+
<ix:nonNumeric contextRef="c-2" name="us-gaap:ScheduleOfBusinessAcquisitionsByAcquisitionTextBlock" id="f-2">
|
|
281
|
+
<p>Activision purchase price {_FILLER}
|
|
282
|
+
<ix:nonFraction name="us-gaap:BusinessCombinationConsiderationTransferred1" contextRef="c-2">61800</ix:nonFraction></p>
|
|
283
|
+
</ix:nonNumeric>
|
|
284
|
+
</body></html>
|
|
285
|
+
"""
|
|
286
|
+
sections = iXBRLParser().parse(html)
|
|
287
|
+
assert len(sections) == 1
|
|
288
|
+
s = sections[0]
|
|
289
|
+
assert s.content.index("Nuance purchase price") < s.content.index(
|
|
290
|
+
"Activision purchase price"
|
|
291
|
+
)
|
|
292
|
+
assert "Other text between" not in s.content
|
|
293
|
+
assert s.xbrl_elements == [
|
|
294
|
+
"us-gaap:BusinessCombinationConsiderationTransferred1",
|
|
295
|
+
"us-gaap:Goodwill",
|
|
296
|
+
]
|
|
297
|
+
|
|
298
|
+
def test_nested_block_of_the_same_name_is_not_repeated(self):
|
|
299
|
+
html = f"""
|
|
300
|
+
<html><body>
|
|
301
|
+
<ix:nonNumeric contextRef="c-1" name="us-gaap:DebtDisclosureTextBlock" id="f-1">
|
|
302
|
+
<p>Outer debt note {_FILLER}</p>
|
|
303
|
+
<ix:nonNumeric contextRef="c-2" name="us-gaap:DebtDisclosureTextBlock" id="f-2">
|
|
304
|
+
<p>Inner repeat {_FILLER}</p>
|
|
305
|
+
</ix:nonNumeric>
|
|
306
|
+
</ix:nonNumeric>
|
|
307
|
+
</body></html>
|
|
308
|
+
"""
|
|
309
|
+
sections = iXBRLParser().parse(html)
|
|
310
|
+
assert len(sections) == 1
|
|
311
|
+
assert sections[0].content.count("Inner repeat") == 1
|
|
312
|
+
|
|
313
|
+
def test_drops_ix_exclude_content(self):
|
|
314
|
+
"""``ix:exclude`` marks content inside a tagged block that is not part
|
|
315
|
+
of the fact — the page header Oracle and Microsoft place inside a policy
|
|
316
|
+
(32 and 50 per filing). Arelle drops it; so must the section."""
|
|
317
|
+
html = f"""
|
|
318
|
+
<html><body>
|
|
319
|
+
<ix:nonNumeric contextRef="c-1" name="us-gaap:IncomeTaxPolicyTextBlock" id="f-1" continuedAt="f-1-a">
|
|
320
|
+
<p>Income taxes are accounted for under ASC 740. {_FILLER}</p>
|
|
321
|
+
<ix:exclude><div><p>80</p><p>Table of Contents</p><p>Index to Financial Statements</p></div></ix:exclude>
|
|
322
|
+
<p>The second step measures the benefit.</p>
|
|
323
|
+
</ix:nonNumeric>
|
|
324
|
+
<ix:continuation id="f-1-a">
|
|
325
|
+
<ix:exclude><p>81 Table of Contents</p></ix:exclude>
|
|
326
|
+
<p>Continued policy text after the page break.</p>
|
|
327
|
+
</ix:continuation>
|
|
328
|
+
</body></html>
|
|
329
|
+
"""
|
|
330
|
+
sections = iXBRLParser().parse(html)
|
|
331
|
+
assert len(sections) == 1
|
|
332
|
+
content = sections[0].content
|
|
333
|
+
assert "Table of Contents" not in content
|
|
334
|
+
assert "Index to Financial Statements" not in content
|
|
335
|
+
assert "The second step measures the benefit." in content
|
|
336
|
+
assert "Continued policy text after the page break." in content
|
|
265
337
|
|
|
266
338
|
def test_multiple_sections(self):
|
|
267
339
|
html = f"""
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""xbrlkit — work with XBRL filings above Arelle: one parse, portable models.
|
|
2
|
+
|
|
3
|
+
Fetch a filing from EDGAR (:mod:`xbrlkit.edgar`), parse it once into the
|
|
4
|
+
neutral :class:`XbrlModel` (:mod:`xbrlkit.parse`), and project that model
|
|
5
|
+
into whichever representation you need (:mod:`xbrlkit.serialize`): the
|
|
6
|
+
``holon.jsonld`` RDF/JSON-LD document, the Project Tavi compiled model,
|
|
7
|
+
xBRL-JSON, or the property-graph tables and a single-filing LadybugDB file.
|
|
8
|
+
:mod:`xbrlkit.text` reads the filing's primary HTML into sections without
|
|
9
|
+
Arelle at all; :mod:`xbrlkit.query` runs SPARQL over a built holon.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
15
|
+
|
|
16
|
+
from .model import XbrlModel
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _get_version() -> str:
|
|
20
|
+
try:
|
|
21
|
+
return version("xbrlkit")
|
|
22
|
+
except PackageNotFoundError:
|
|
23
|
+
return "0.0.0+development"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
__version__ = _get_version()
|
|
27
|
+
|
|
28
|
+
__all__ = ("XbrlModel", "__version__")
|
|
@@ -264,6 +264,10 @@ class Network(BaseModel):
|
|
|
264
264
|
|
|
265
265
|
role_uri: str
|
|
266
266
|
definition: str | None = None
|
|
267
|
+
# A second, longer reading of the role — XBRL's documentation label role.
|
|
268
|
+
# A producer whose definition is a composed sort key ("0001 - Statement -
|
|
269
|
+
# Balance Sheet") keeps the role's own name here, so it is not lost.
|
|
270
|
+
documentation: str | None = None
|
|
267
271
|
kind: NetworkKind
|
|
268
272
|
arcs: list[Arc] = Field(default_factory=list)
|
|
269
273
|
# The ``id`` of the role's ``<link:roleType>`` in the filer's schema, when
|
|
@@ -3,27 +3,41 @@
|
|
|
3
3
|
xBRL-JSON (REC 2021) and Tavi (PWD 2026-09-01) are two serialisations of the
|
|
4
4
|
same Open Information Model, and the literal forms below are where they agree:
|
|
5
5
|
a period is an ISO 8601 interval of ``xs:dateTime`` values with an exclusive
|
|
6
|
-
end, a language tag is lower case, and
|
|
7
|
-
the ``cik`` prefix
|
|
8
|
-
|
|
9
|
-
one be checked against
|
|
6
|
+
end, a language tag is lower case, and an entity is written scheme-first: the
|
|
7
|
+
SEC scheme under the ``cik`` prefix, any other under ``entity``. Keeping them
|
|
8
|
+
in one place means a fact's period or entity reads identically in both
|
|
9
|
+
projections of one filing — which is also what lets one be checked against
|
|
10
|
+
the other.
|
|
10
11
|
"""
|
|
11
12
|
|
|
12
13
|
from __future__ import annotations
|
|
13
14
|
|
|
14
15
|
from datetime import date, timedelta
|
|
15
16
|
|
|
16
|
-
from ..model import Period
|
|
17
|
+
from ..model import EntityIdentity, Period
|
|
17
18
|
|
|
18
19
|
# The SEC's entity scheme, bound to the prefix Arelle uses for it. An entity is
|
|
19
20
|
# identified by scheme + identifier, so the scheme is the namespace and the
|
|
20
21
|
# CIK the local name: ``cik:0000066740``.
|
|
21
22
|
CIK_PREFIX = "cik"
|
|
22
23
|
CIK_SCHEME = "http://www.sec.gov/CIK"
|
|
24
|
+
# Any other scheme binds under a neutral prefix. A model that did not come from
|
|
25
|
+
# an EDGAR filing — a ledger's own report, say — still identifies its entity by
|
|
26
|
+
# scheme + identifier, and writes it ``entity:<identifier>``.
|
|
27
|
+
ENTITY_PREFIX = "entity"
|
|
23
28
|
|
|
24
29
|
|
|
25
|
-
def
|
|
26
|
-
|
|
30
|
+
def entity_prefix(entity: EntityIdentity) -> tuple[str, str]:
|
|
31
|
+
"""The ``(prefix, scheme)`` pair an entity's SQName is bound under."""
|
|
32
|
+
if entity.scheme == CIK_SCHEME:
|
|
33
|
+
return CIK_PREFIX, CIK_SCHEME
|
|
34
|
+
return ENTITY_PREFIX, entity.scheme
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def entity_sqname(entity: EntityIdentity) -> str:
|
|
38
|
+
"""``cik:0000066740`` for an SEC filer; ``entity:<id>`` under any other scheme."""
|
|
39
|
+
prefix, _ = entity_prefix(entity)
|
|
40
|
+
return f"{prefix}:{entity.cik}"
|
|
27
41
|
|
|
28
42
|
|
|
29
43
|
def period_interval(period: Period) -> str:
|
|
@@ -28,8 +28,7 @@ import json
|
|
|
28
28
|
|
|
29
29
|
from ..model import Concept, XbrlModel
|
|
30
30
|
from ._values import (
|
|
31
|
-
|
|
32
|
-
CIK_SCHEME,
|
|
31
|
+
entity_prefix,
|
|
33
32
|
entity_sqname,
|
|
34
33
|
language_tag,
|
|
35
34
|
period_interval,
|
|
@@ -45,7 +44,7 @@ OIM_RESERVED_NAMESPACES: dict[str, str] = {
|
|
|
45
44
|
"utr": "http://www.xbrl.org/2009/utr",
|
|
46
45
|
}
|
|
47
46
|
|
|
48
|
-
# The
|
|
47
|
+
# The entity SQName and its scheme binding, the period literal and the
|
|
49
48
|
# language form are shared with the Tavi projection — see ``_values``.
|
|
50
49
|
|
|
51
50
|
# Unit measures that mean "no unit" and are therefore left off the fact.
|
|
@@ -75,7 +74,8 @@ def to_oim_document(
|
|
|
75
74
|
def _namespaces(model: XbrlModel) -> dict[str, str]:
|
|
76
75
|
"""Prefix map: the reserved set, the entity scheme, and the filing's own."""
|
|
77
76
|
namespaces = dict(OIM_RESERVED_NAMESPACES)
|
|
78
|
-
|
|
77
|
+
prefix, scheme = entity_prefix(model.entity)
|
|
78
|
+
namespaces[prefix] = scheme
|
|
79
79
|
|
|
80
80
|
by_uri = {uri: prefix for prefix, uri in namespaces.items()}
|
|
81
81
|
for concept in model.concepts.values():
|
|
@@ -95,7 +95,7 @@ def _facts(model: XbrlModel, namespaces: dict[str, str]) -> dict[str, object]:
|
|
|
95
95
|
prefixes = {uri: prefix for prefix, uri in namespaces.items()}
|
|
96
96
|
periods = {period.id: period for period in model.periods}
|
|
97
97
|
units = {unit.id: unit for unit in model.units}
|
|
98
|
-
entity = entity_sqname(model.entity
|
|
98
|
+
entity = entity_sqname(model.entity)
|
|
99
99
|
facts: dict[str, object] = {}
|
|
100
100
|
concepts = model.concepts
|
|
101
101
|
|
|
@@ -51,8 +51,7 @@ from dataclasses import dataclass, field
|
|
|
51
51
|
from ..model import Arc, Concept, Network, Period, XbrlFact, XbrlModel
|
|
52
52
|
from ..namespaces import TAVI_REPORT_BASE
|
|
53
53
|
from ._values import (
|
|
54
|
-
|
|
55
|
-
CIK_SCHEME,
|
|
54
|
+
entity_prefix,
|
|
56
55
|
entity_sqname,
|
|
57
56
|
language_tag,
|
|
58
57
|
period_interval,
|
|
@@ -206,6 +205,7 @@ LABEL_ROLE_TYPES: dict[str, str] = {
|
|
|
206
205
|
"http://xbrl.us/us-gaap/role/label/negatedTotal": "xbrl:negatedTotal",
|
|
207
206
|
}
|
|
208
207
|
DEFAULT_LABEL_TYPE = "xbrl:label"
|
|
208
|
+
DOCUMENTATION_LABEL_TYPE = "xbrl:documentation"
|
|
209
209
|
|
|
210
210
|
PRESENTATION_RELATIONSHIP = "xbrl:parent-child"
|
|
211
211
|
CALCULATION_RELATIONSHIP = "xbrl:summation-item"
|
|
@@ -401,16 +401,23 @@ class GapReport:
|
|
|
401
401
|
}
|
|
402
402
|
|
|
403
403
|
|
|
404
|
-
def to_tavi(
|
|
404
|
+
def to_tavi(
|
|
405
|
+
model: XbrlModel, *, report_id: str | None = None, description: str | None = None
|
|
406
|
+
) -> str:
|
|
405
407
|
"""Project ``model`` into a Tavi compiled-model JSON string."""
|
|
406
|
-
document, _ = to_tavi_report(model, report_id=report_id)
|
|
408
|
+
document, _ = to_tavi_report(model, report_id=report_id, description=description)
|
|
407
409
|
return json.dumps(document, indent=2, sort_keys=False, default=str)
|
|
408
410
|
|
|
409
411
|
|
|
410
412
|
def to_tavi_report(
|
|
411
|
-
model: XbrlModel, *, report_id: str | None = None
|
|
413
|
+
model: XbrlModel, *, report_id: str | None = None, description: str | None = None
|
|
412
414
|
) -> tuple[dict[str, object], GapReport]:
|
|
413
|
-
"""Project ``model``, returning the document and what it could not express.
|
|
415
|
+
"""Project ``model``, returning the document and what it could not express.
|
|
416
|
+
|
|
417
|
+
``report_id`` scopes the report's own namespace (the accession by default);
|
|
418
|
+
``description`` replaces the ``documentInfo`` sentence, which otherwise
|
|
419
|
+
reads the model as an EDGAR filing.
|
|
420
|
+
"""
|
|
414
421
|
report_id = report_id or model.filing.accession
|
|
415
422
|
gaps = GapReport()
|
|
416
423
|
namespaces = _namespaces(model, report_id)
|
|
@@ -439,7 +446,9 @@ def to_tavi_report(
|
|
|
439
446
|
"domainNetworks": dimensional.domain_networks,
|
|
440
447
|
"members": dimensional.members,
|
|
441
448
|
"cubes": dimensional.cubes,
|
|
442
|
-
"labels": _labels(model, gaps, default_language)
|
|
449
|
+
"labels": _labels(model, gaps, default_language)
|
|
450
|
+
+ _entity_labels(model, default_language)
|
|
451
|
+
+ group_labels,
|
|
443
452
|
"networks": networks,
|
|
444
453
|
"groups": groups,
|
|
445
454
|
"groupContents": group_contents,
|
|
@@ -450,7 +459,8 @@ def to_tavi_report(
|
|
|
450
459
|
"documentInfo": {
|
|
451
460
|
"documentType": DOCTYPE_COMPILED,
|
|
452
461
|
"namespaces": namespaces,
|
|
453
|
-
"description":
|
|
462
|
+
"description": description
|
|
463
|
+
or (
|
|
454
464
|
f"{model.filing.form or 'filing'} {model.filing.accession} "
|
|
455
465
|
f"(CIK {model.filing.cik}) projected from XBRL by xbrlkit"
|
|
456
466
|
),
|
|
@@ -703,8 +713,10 @@ def _namespaces(model: XbrlModel, report_id: str) -> dict[str, str]:
|
|
|
703
713
|
"""
|
|
704
714
|
namespaces = dict(RESERVED_NAMESPACES)
|
|
705
715
|
namespaces[REPORT_PREFIX] = _report_namespace(report_id)
|
|
706
|
-
# The entity scheme: the SQName ``cik:0000066740``
|
|
707
|
-
|
|
716
|
+
# The entity scheme: the SQName ``cik:0000066740`` (or ``entity:<id>`` under
|
|
717
|
+
# any other scheme) needs it bound.
|
|
718
|
+
prefix, scheme = entity_prefix(model.entity)
|
|
719
|
+
namespaces[prefix] = scheme
|
|
708
720
|
|
|
709
721
|
by_uri = {uri: prefix for prefix, uri in namespaces.items()}
|
|
710
722
|
for concept in model.concepts.values():
|
|
@@ -761,13 +773,33 @@ def _default_language(model: XbrlModel) -> str:
|
|
|
761
773
|
def _entities(model: XbrlModel) -> list[dict[str, object]]:
|
|
762
774
|
"""The reporting entity (section 8.1).
|
|
763
775
|
|
|
764
|
-
An entity's SQName "includes the scheme and the identifier", so the
|
|
765
|
-
|
|
766
|
-
same name the OIM projection writes and Arelle's converter
|
|
767
|
-
|
|
768
|
-
|
|
776
|
+
An entity's SQName "includes the scheme and the identifier", so the scheme
|
|
777
|
+
is the namespace and the identifier the local name — ``cik:0000066740`` for
|
|
778
|
+
an SEC filer, the same name the OIM projection writes and Arelle's converter
|
|
779
|
+
emits, and ``entity:<id>`` under any other scheme. It was previously minted
|
|
780
|
+
under this report's own namespace, which dropped the scheme and gave two
|
|
781
|
+
converters of one filing two different entities.
|
|
769
782
|
"""
|
|
770
|
-
return [{"name": entity_sqname(model.entity
|
|
783
|
+
return [{"name": entity_sqname(model.entity)}]
|
|
784
|
+
|
|
785
|
+
|
|
786
|
+
def _entity_labels(model: XbrlModel, default_language: str) -> list[dict[str, object]]:
|
|
787
|
+
"""The entity's name as a label object pointing at the entity (section 5.14).
|
|
788
|
+
|
|
789
|
+
``forObject`` is "any", and a reader holding the compiled model and nothing
|
|
790
|
+
else — no ``dei`` facts, no EDGAR header — still needs a name to put on the
|
|
791
|
+
report. The entity object itself carries only its SQName.
|
|
792
|
+
"""
|
|
793
|
+
if not model.entity.name:
|
|
794
|
+
return []
|
|
795
|
+
return [
|
|
796
|
+
{
|
|
797
|
+
"forObject": entity_sqname(model.entity),
|
|
798
|
+
"labelType": DEFAULT_LABEL_TYPE,
|
|
799
|
+
"value": model.entity.name,
|
|
800
|
+
"language": default_language,
|
|
801
|
+
}
|
|
802
|
+
]
|
|
771
803
|
|
|
772
804
|
|
|
773
805
|
def _units(model: XbrlModel) -> list[dict[str, object]]:
|
|
@@ -995,9 +1027,12 @@ def _networks_and_groups(
|
|
|
995
1027
|
group_labels: list[dict[str, object]] = []
|
|
996
1028
|
group_names: dict[str, str] = {}
|
|
997
1029
|
definitions: dict[str, str] = {}
|
|
1030
|
+
documentations: dict[str, str] = {}
|
|
998
1031
|
for network in model.networks:
|
|
999
1032
|
if network.definition and network.role_uri not in definitions:
|
|
1000
1033
|
definitions[network.role_uri] = network.definition
|
|
1034
|
+
if network.documentation and network.role_uri not in documentations:
|
|
1035
|
+
documentations[network.role_uri] = network.documentation
|
|
1001
1036
|
|
|
1002
1037
|
def group_for(role_uri: str) -> str:
|
|
1003
1038
|
group_name = group_names.get(role_uri)
|
|
@@ -1019,6 +1054,16 @@ def _networks_and_groups(
|
|
|
1019
1054
|
"language": default_language,
|
|
1020
1055
|
}
|
|
1021
1056
|
)
|
|
1057
|
+
documentation = documentations.get(role_uri)
|
|
1058
|
+
if documentation:
|
|
1059
|
+
group_labels.append(
|
|
1060
|
+
{
|
|
1061
|
+
"forObject": group_name,
|
|
1062
|
+
"labelType": DOCUMENTATION_LABEL_TYPE,
|
|
1063
|
+
"value": documentation,
|
|
1064
|
+
"language": default_language,
|
|
1065
|
+
}
|
|
1066
|
+
)
|
|
1022
1067
|
return group_name
|
|
1023
1068
|
|
|
1024
1069
|
for index, network in enumerate(model.networks):
|
|
@@ -1089,7 +1134,7 @@ def _facts(
|
|
|
1089
1134
|
"""
|
|
1090
1135
|
periods = {period.id: period for period in model.periods}
|
|
1091
1136
|
units = {unit.id: unit for unit in model.units}
|
|
1092
|
-
entity = entity_sqname(model.entity
|
|
1137
|
+
entity = entity_sqname(model.entity)
|
|
1093
1138
|
facts: list[dict[str, object]] = []
|
|
1094
1139
|
|
|
1095
1140
|
for index, fact in enumerate(model.facts):
|
|
@@ -26,6 +26,12 @@ Continuations nest: a table text block inside a revenue note continues in a
|
|
|
26
26
|
``ix:continuation`` that sits physically inside the note's own continuation.
|
|
27
27
|
Every continuation is registered with its offsets, nested or not, and a chain
|
|
28
28
|
appends a link only when no appended link already contains it.
|
|
29
|
+
|
|
30
|
+
A concept tagged more than once — a purchase-price table per acquisition, a
|
|
31
|
+
roll-forward per period — is one section holding every distinct occurrence
|
|
32
|
+
in document order; keeping the first alone lost the second table. Content
|
|
33
|
+
inside ``ix:exclude`` (the page header some filers place inside a tagged
|
|
34
|
+
block) is not part of the fact and is dropped, as Arelle drops it.
|
|
29
35
|
"""
|
|
30
36
|
|
|
31
37
|
from __future__ import annotations
|
|
@@ -152,6 +158,12 @@ class _Block:
|
|
|
152
158
|
|
|
153
159
|
_TAG_NAME_END = " \t\r\n/>"
|
|
154
160
|
|
|
161
|
+
# Content a filer marks as not part of the fact: page headers and footers
|
|
162
|
+
# inside a tagged block. Applied to one section's HTML, never the document.
|
|
163
|
+
_EXCLUDE_RE = re.compile(
|
|
164
|
+
r"<ix:exclude\b[^>]*>.*?</ix:exclude\s*>", re.IGNORECASE | re.DOTALL
|
|
165
|
+
)
|
|
166
|
+
|
|
155
167
|
|
|
156
168
|
def _scan_blocks(html: str, tag: str) -> list[_Block]:
|
|
157
169
|
"""Every ``<tag>`` block in document order, nested blocks included.
|
|
@@ -222,14 +234,21 @@ class iXBRLParser: # noqa: N801 — the established name across consumers
|
|
|
222
234
|
``ix:continuation`` chain resolved, its numeric facts listed — nested
|
|
223
235
|
ones included: a policy block inside an accounting-policies note is its
|
|
224
236
|
own section as well as part of the note's text, because its continued
|
|
225
|
-
tail belongs to its own chain and to no other. A
|
|
226
|
-
|
|
227
|
-
|
|
237
|
+
tail belongs to its own chain and to no other. A concept tagged more
|
|
238
|
+
than once is one section holding every distinct occurrence in document
|
|
239
|
+
order (the same table tagged twice with different markup counts once);
|
|
240
|
+
``ix:exclude`` content is dropped. A section over ``part_size`` is
|
|
241
|
+
returned as consecutive parts that share the ``section_id`` and carry
|
|
242
|
+
``part``/``part_count``.
|
|
228
243
|
"""
|
|
229
244
|
continuations = self._build_continuation_map(html)
|
|
230
245
|
|
|
231
|
-
|
|
232
|
-
|
|
246
|
+
# Per element name, in document order: each distinct occurrence's HTML
|
|
247
|
+
# and text, the blocks already taken, and the texts already seen.
|
|
248
|
+
htmls: dict[str, list[str]] = {}
|
|
249
|
+
texts: dict[str, list[str]] = {}
|
|
250
|
+
taken: dict[str, list[_Block]] = {}
|
|
251
|
+
seen: dict[str, set[str]] = {}
|
|
233
252
|
|
|
234
253
|
for block in _scan_blocks(html, "ix:nonNumeric"):
|
|
235
254
|
name_match = re.search(
|
|
@@ -243,10 +262,9 @@ class iXBRLParser: # noqa: N801 — the established name across consumers
|
|
|
243
262
|
if element_name.startswith(("dei:", "ecd:")):
|
|
244
263
|
continue
|
|
245
264
|
|
|
246
|
-
#
|
|
247
|
-
if
|
|
265
|
+
# A block nested in an earlier block of the same name is already in it
|
|
266
|
+
if any(outer.contains(block) for outer in taken.get(element_name, ())):
|
|
248
267
|
continue
|
|
249
|
-
seen_ids.add(element_name)
|
|
250
268
|
|
|
251
269
|
full_html = block.content(html)
|
|
252
270
|
continued_at = _continued_at(block.attrs)
|
|
@@ -254,14 +272,25 @@ class iXBRLParser: # noqa: N801 — the established name across consumers
|
|
|
254
272
|
full_html += self._resolve_continuation_chain(
|
|
255
273
|
continued_at, continuations, html, covered=[block]
|
|
256
274
|
)
|
|
257
|
-
|
|
258
|
-
xbrl_elements = _extract_elements_from_block(full_html)
|
|
275
|
+
full_html = _EXCLUDE_RE.sub(" ", full_html)
|
|
259
276
|
content = _strip_html(full_html)
|
|
260
277
|
|
|
278
|
+
key = " ".join(content.split())
|
|
279
|
+
if key in seen.setdefault(element_name, set()):
|
|
280
|
+
continue
|
|
281
|
+
seen[element_name].add(key)
|
|
282
|
+
taken.setdefault(element_name, []).append(block)
|
|
283
|
+
htmls.setdefault(element_name, []).append(full_html)
|
|
284
|
+
texts.setdefault(element_name, []).append(content)
|
|
285
|
+
|
|
286
|
+
sections: list[iXBRLSection] = []
|
|
287
|
+
for element_name, occurrences in texts.items():
|
|
288
|
+
content = "\n\n".join(occurrences)
|
|
261
289
|
word_count = len(content.split())
|
|
262
290
|
if word_count < MIN_SECTION_WORDS:
|
|
263
291
|
continue
|
|
264
292
|
|
|
293
|
+
xbrl_elements = _extract_elements_from_block("".join(htmls[element_name]))
|
|
265
294
|
section_label = _label_from_element_name(element_name)
|
|
266
295
|
parts = split_text(content, self.part_size)
|
|
267
296
|
for index, part_text in enumerate(parts, start=1):
|
|
@@ -281,7 +310,7 @@ class iXBRLParser: # noqa: N801 — the established name across consumers
|
|
|
281
310
|
logger.debug(
|
|
282
311
|
"iXBRL parsed: %d disclosure documents (%d sections), %d elements",
|
|
283
312
|
len(sections),
|
|
284
|
-
len(
|
|
313
|
+
len(texts),
|
|
285
314
|
sum(s.element_count for s in sections if s.part == 1),
|
|
286
315
|
)
|
|
287
316
|
return sections
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
"""xbrlkit — a SEC XBRL filing to a portable ``holon.jsonld``.
|
|
2
|
-
|
|
3
|
-
Fetch a filing from EDGAR, parse it into the neutral :class:`XbrlModel`, and
|
|
4
|
-
project that into the canonical scene/boundary/projection holon that renders
|
|
5
|
-
offline in the RoboSystems holon viewer.
|
|
6
|
-
"""
|
|
7
|
-
|
|
8
|
-
from __future__ import annotations
|
|
9
|
-
|
|
10
|
-
from importlib.metadata import PackageNotFoundError, version
|
|
11
|
-
|
|
12
|
-
from .model import XbrlModel
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
def _get_version() -> str:
|
|
16
|
-
try:
|
|
17
|
-
return version("xbrlkit")
|
|
18
|
-
except PackageNotFoundError:
|
|
19
|
-
return "0.0.0+development"
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
__version__ = _get_version()
|
|
23
|
-
|
|
24
|
-
__all__ = ("XbrlModel", "__version__")
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/conf/README.md
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/conf/saxon9.jar
RENAMED
|
File without changes
|
{xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/conf/testcase.xml
RENAMED
|
File without changes
|
{xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/conf/tests.xml
RENAMED
|
File without changes
|
{xbrlkit-0.4.0 → xbrlkit-0.4.2}/xbrlkit/_vendor/arelle_plugins/EDGAR/transform/conf/tests.xsd
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|