larc-iudex 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. larc_iudex-0.1.0/LICENSE +21 -0
  2. larc_iudex-0.1.0/PKG-INFO +243 -0
  3. larc_iudex-0.1.0/README.md +203 -0
  4. larc_iudex-0.1.0/iudex/__init__.py +22 -0
  5. larc_iudex-0.1.0/iudex/__main__.py +96 -0
  6. larc_iudex-0.1.0/iudex/common/__init__.py +0 -0
  7. larc_iudex-0.1.0/iudex/common/log.py +65 -0
  8. larc_iudex-0.1.0/iudex/common/training.py +613 -0
  9. larc_iudex-0.1.0/iudex/rst/__init__.py +34 -0
  10. larc_iudex-0.1.0/iudex/rst/data/__init__.py +0 -0
  11. larc_iudex-0.1.0/iudex/rst/data/metrics.py +280 -0
  12. larc_iudex-0.1.0/iudex/rst/data/reader.py +280 -0
  13. larc_iudex-0.1.0/iudex/rst/data/seg_metrics.py +157 -0
  14. larc_iudex-0.1.0/iudex/rst/data/tree.py +1106 -0
  15. larc_iudex-0.1.0/iudex/rst/parsers/__init__.py +94 -0
  16. larc_iudex-0.1.0/iudex/rst/parsers/common/__init__.py +0 -0
  17. larc_iudex-0.1.0/iudex/rst/parsers/common/biaffine.py +49 -0
  18. larc_iudex-0.1.0/iudex/rst/parsers/common/config.py +135 -0
  19. larc_iudex-0.1.0/iudex/rst/parsers/common/curriculum.py +144 -0
  20. larc_iudex-0.1.0/iudex/rst/parsers/common/detokenization.py +43 -0
  21. larc_iudex-0.1.0/iudex/rst/parsers/common/encoding.py +413 -0
  22. larc_iudex-0.1.0/iudex/rst/parsers/common/generative_eval.py +232 -0
  23. larc_iudex-0.1.0/iudex/rst/parsers/common/inference.py +86 -0
  24. larc_iudex-0.1.0/iudex/rst/parsers/common/pointer.py +47 -0
  25. larc_iudex-0.1.0/iudex/rst/parsers/common/predict_cli.py +152 -0
  26. larc_iudex-0.1.0/iudex/rst/parsers/common/segmentation.py +300 -0
  27. larc_iudex-0.1.0/iudex/rst/parsers/common/seqgen.py +522 -0
  28. larc_iudex-0.1.0/iudex/rst/parsers/common/sexp_constraints.py +763 -0
  29. larc_iudex-0.1.0/iudex/rst/parsers/dmrst/__init__.py +2 -0
  30. larc_iudex-0.1.0/iudex/rst/parsers/dmrst/configuration_dmrst.py +169 -0
  31. larc_iudex-0.1.0/iudex/rst/parsers/dmrst/modeling_dmrst.py +733 -0
  32. larc_iudex-0.1.0/iudex/rst/parsers/dmrst/predict_dmrst.py +9 -0
  33. larc_iudex-0.1.0/iudex/rst/parsers/dmrst/train_dmrst.py +528 -0
  34. larc_iudex-0.1.0/iudex/rst/parsers/gen/__init__.py +4 -0
  35. larc_iudex-0.1.0/iudex/rst/parsers/gen/backbones/__init__.py +10 -0
  36. larc_iudex-0.1.0/iudex/rst/parsers/gen/backbones/base.py +466 -0
  37. larc_iudex-0.1.0/iudex/rst/parsers/gen/backbones/decoder_only.py +415 -0
  38. larc_iudex-0.1.0/iudex/rst/parsers/gen/backbones/seq2seq.py +372 -0
  39. larc_iudex-0.1.0/iudex/rst/parsers/gen/configuration_gen.py +299 -0
  40. larc_iudex-0.1.0/iudex/rst/parsers/gen/decode.py +337 -0
  41. larc_iudex-0.1.0/iudex/rst/parsers/gen/errors.py +31 -0
  42. larc_iudex-0.1.0/iudex/rst/parsers/gen/eval_gen.py +176 -0
  43. larc_iudex-0.1.0/iudex/rst/parsers/gen/modeling_gen.py +384 -0
  44. larc_iudex-0.1.0/iudex/rst/parsers/gen/predict_gen.py +9 -0
  45. larc_iudex-0.1.0/iudex/rst/parsers/gen/serializations/__init__.py +10 -0
  46. larc_iudex-0.1.0/iudex/rst/parsers/gen/serializations/base.py +231 -0
  47. larc_iudex-0.1.0/iudex/rst/parsers/gen/serializations/sexp.py +487 -0
  48. larc_iudex-0.1.0/iudex/rst/parsers/gen/serializations/sr.py +351 -0
  49. larc_iudex-0.1.0/iudex/rst/parsers/gen/train_gen.py +484 -0
  50. larc_iudex-0.1.0/iudex/rst/parsers/hfhub/__init__.py +22 -0
  51. larc_iudex-0.1.0/iudex/rst/parsers/hfhub/datasets.py +81 -0
  52. larc_iudex-0.1.0/iudex/rst/parsers/hfhub/hub.py +455 -0
  53. larc_iudex-0.1.0/iudex/rst/parsers/hfhub/push.py +49 -0
  54. larc_iudex-0.1.0/iudex/rst/parsers/icl/__init__.py +14 -0
  55. larc_iudex-0.1.0/iudex/rst/parsers/icl/_cli_common.py +48 -0
  56. larc_iudex-0.1.0/iudex/rst/parsers/icl/configuration_icl.py +274 -0
  57. larc_iudex-0.1.0/iudex/rst/parsers/icl/eval_icl.py +239 -0
  58. larc_iudex-0.1.0/iudex/rst/parsers/icl/eval_metrics.py +95 -0
  59. larc_iudex-0.1.0/iudex/rst/parsers/icl/modeling_icl.py +293 -0
  60. larc_iudex-0.1.0/iudex/rst/parsers/icl/predict_icl.py +84 -0
  61. larc_iudex-0.1.0/iudex/rst/parsers/icl/provider.py +445 -0
  62. larc_iudex-0.1.0/iudex/rst/parsers/icl/serialize.py +748 -0
  63. larc_iudex-0.1.0/iudex/rst/parsers/sr_biaffine/__init__.py +2 -0
  64. larc_iudex-0.1.0/iudex/rst/parsers/sr_biaffine/configuration_sr_biaffine.py +82 -0
  65. larc_iudex-0.1.0/iudex/rst/parsers/sr_biaffine/modeling_sr_biaffine.py +240 -0
  66. larc_iudex-0.1.0/iudex/rst/parsers/sr_biaffine/predict_sr_biaffine.py +9 -0
  67. larc_iudex-0.1.0/iudex/rst/parsers/sr_biaffine/train_sr_biaffine.py +369 -0
  68. larc_iudex-0.1.0/iudex/rst/parsers/topdown_biaffine/__init__.py +2 -0
  69. larc_iudex-0.1.0/iudex/rst/parsers/topdown_biaffine/configuration_topdown_biaffine.py +76 -0
  70. larc_iudex-0.1.0/iudex/rst/parsers/topdown_biaffine/modeling_topdown_biaffine.py +216 -0
  71. larc_iudex-0.1.0/iudex/rst/parsers/topdown_biaffine/predict_topdown_biaffine.py +9 -0
  72. larc_iudex-0.1.0/iudex/rst/parsers/topdown_biaffine/train_topdown_biaffine.py +369 -0
  73. larc_iudex-0.1.0/iudex/runs.py +745 -0
  74. larc_iudex-0.1.0/larc_iudex.egg-info/PKG-INFO +243 -0
  75. larc_iudex-0.1.0/larc_iudex.egg-info/SOURCES.txt +113 -0
  76. larc_iudex-0.1.0/larc_iudex.egg-info/dependency_links.txt +1 -0
  77. larc_iudex-0.1.0/larc_iudex.egg-info/entry_points.txt +2 -0
  78. larc_iudex-0.1.0/larc_iudex.egg-info/requires.txt +16 -0
  79. larc_iudex-0.1.0/larc_iudex.egg-info/top_level.txt +1 -0
  80. larc_iudex-0.1.0/pyproject.toml +84 -0
  81. larc_iudex-0.1.0/setup.cfg +4 -0
  82. larc_iudex-0.1.0/tests/test_action_loss_chunked.py +106 -0
  83. larc_iudex-0.1.0/tests/test_beam_nan_poisoning.py +95 -0
  84. larc_iudex-0.1.0/tests/test_budget_range_gold_edu_forcer.py +369 -0
  85. larc_iudex-0.1.0/tests/test_decode_invariants.py +210 -0
  86. larc_iudex-0.1.0/tests/test_decode_positional_budget.py +160 -0
  87. larc_iudex-0.1.0/tests/test_edu_token_alignment.py +139 -0
  88. larc_iudex-0.1.0/tests/test_encoder_backbones.py +274 -0
  89. larc_iudex-0.1.0/tests/test_gen_accum_windows.py +143 -0
  90. larc_iudex-0.1.0/tests/test_gen_batched_greedy_equivalence.py +180 -0
  91. larc_iudex-0.1.0/tests/test_gen_batched_sexp_cascade.py +171 -0
  92. larc_iudex-0.1.0/tests/test_gen_document_weights.py +39 -0
  93. larc_iudex-0.1.0/tests/test_gen_equivalence.py +191 -0
  94. larc_iudex-0.1.0/tests/test_gen_inference_method.py +44 -0
  95. larc_iudex-0.1.0/tests/test_gen_overlength.py +90 -0
  96. larc_iudex-0.1.0/tests/test_gen_train_smoke.py +165 -0
  97. larc_iudex-0.1.0/tests/test_gold_edu_beam.py +183 -0
  98. larc_iudex-0.1.0/tests/test_gold_edu_semantics_parity.py +96 -0
  99. larc_iudex-0.1.0/tests/test_icl.py +491 -0
  100. larc_iudex-0.1.0/tests/test_peft_inert_fields.py +116 -0
  101. larc_iudex-0.1.0/tests/test_qlora_4bit_loading.py +215 -0
  102. larc_iudex-0.1.0/tests/test_retired_config_keys.py +66 -0
  103. larc_iudex-0.1.0/tests/test_seq2seq_input_budget.py +190 -0
  104. larc_iudex-0.1.0/tests/test_seqgen_checkpoint_and_batch.py +113 -0
  105. larc_iudex-0.1.0/tests/test_sexp_constraints.py +712 -0
  106. larc_iudex-0.1.0/tests/test_sexp_empty_tree_fallback.py +101 -0
  107. larc_iudex-0.1.0/tests/test_sexp_roundtrip.py +100 -0
  108. larc_iudex-0.1.0/tests/test_sexp_words_labels.py +176 -0
  109. larc_iudex-0.1.0/tests/test_shift_reduce_roundtrip.py +141 -0
  110. larc_iudex-0.1.0/tests/test_sr_biaffine.py +108 -0
  111. larc_iudex-0.1.0/tests/test_sr_decode_state.py +105 -0
  112. larc_iudex-0.1.0/tests/test_sr_deep_tree_degrades.py +63 -0
  113. larc_iudex-0.1.0/tests/test_trainable_only_shadow_checkpoint.py +211 -0
  114. larc_iudex-0.1.0/tests/test_training_checkpoint.py +181 -0
  115. larc_iudex-0.1.0/tests/test_wsd_scheduler.py +62 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Luke Gessler
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,243 @@
1
+ Metadata-Version: 2.4
2
+ Name: larc-iudex
3
+ Version: 0.1.0
4
+ Summary: A collection of discourse parsers and associated code focused on readability and usability.
5
+ Author-email: Luke Gessler <lukegessler@gmail.com>
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/larc-iu/iudex
8
+ Project-URL: Repository, https://github.com/larc-iu/iudex
9
+ Project-URL: Issues, https://github.com/larc-iu/iudex/issues
10
+ Keywords: discourse parsing,rhetorical structure theory,RST,NLP,computational linguistics
11
+ Classifier: Development Status :: 3 - Alpha
12
+ Classifier: Intended Audience :: Science/Research
13
+ Classifier: License :: OSI Approved :: MIT License
14
+ Classifier: Operating System :: OS Independent
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.10
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
20
+ Classifier: Topic :: Text Processing :: Linguistic
21
+ Requires-Python: >=3.10
22
+ Description-Content-Type: text/markdown
23
+ License-File: LICENSE
24
+ Requires-Dist: torch
25
+ Requires-Dist: transformers
26
+ Requires-Dist: huggingface_hub
27
+ Requires-Dist: tonga-config
28
+ Requires-Dist: lxml
29
+ Requires-Dist: rich
30
+ Requires-Dist: sentencepiece
31
+ Requires-Dist: sacremoses
32
+ Requires-Dist: peft
33
+ Requires-Dist: tensorboard
34
+ Requires-Dist: setuptools<81
35
+ Provides-Extra: dev
36
+ Requires-Dist: ruff; extra == "dev"
37
+ Requires-Dist: build; extra == "dev"
38
+ Requires-Dist: twine; extra == "dev"
39
+ Dynamic: license-file
40
+
41
+ # IUDEX
42
+
43
+ The **<u>I</u>ndiana <u>U</u>niversity <u>D</u>iscourse <u>Ex</u>hibition** (IUDEX) is a collection of parsers and other code related to discourse parsing.
44
+
45
+ ## Setup
46
+
47
+ For the latest release:
48
+
49
+ ```
50
+ pip install larc-iudex
51
+ ```
52
+
53
+ For the current state of `master`:
54
+
55
+ ```
56
+ pip install git+https://github.com/larc-iu/iudex
57
+ ```
58
+
59
+ Or for development:
60
+
61
+ ```
62
+ git clone https://github.com/larc-iu/iudex && cd iudex
63
+ pip install -e .
64
+ ```
65
+
66
+ Note that the command you will invoke is `iudex`, not `larc-iudex`.
67
+
68
+ ## Quick Start with Inference
69
+
70
+ Parse a sample document end-to-end with a pretrained DMRST model pulled from the HuggingFace Hub. From the command line:
71
+
72
+ ```bash
73
+ iudex dmrst predict \
74
+ --hub-id larc-iu/dmrst-gum-12.1.0 \
75
+ --text "Although the experiment was carefully designed, the results were inconclusive. We plan to repeat it tonight."
76
+ ```
77
+ This yields the parsed tree in `.rs3` format printed to `stdout`:
78
+ ```xml
79
+ <rst>
80
+ <relations><!-- ... --></relations>
81
+ <body>
82
+ <segment id="1" parent="2" relname="adversative-concession">Although the experiment was carefully # designed,</segment>
83
+ <segment id="2" parent="4" relname="span">the results were inconclusive.</segment>
84
+ <segment id="3" parent="5" relname="span">We plan to repeat it tonight.</segment>
85
+ <group id="4" type="span" parent="3" relname="adversative-antithesis"/>
86
+ <group id="5" type="span"/>
87
+ </body>
88
+ </rst>
89
+ ```
90
+
91
+ The same flow from Python:
92
+
93
+ ```python
94
+ from iudex.rst.parsers.dmrst.modeling_dmrst import DMRSTParser
95
+ parser = DMRSTParser.from_pretrained("larc-iu/dmrst-gum-12.1.0")
96
+ tree = parser.predict_from_text(
97
+ "Although the experiment was carefully designed, "
98
+ "the results were inconclusive. "
99
+ "We plan to repeat it tonight."
100
+ )
101
+ print(tree.to_rs4_string())
102
+ ```
103
+ Yields:
104
+ ```xml
105
+ <rst>
106
+ <relations><!-- ... --></relations>
107
+ <body>
108
+ <segment id="1" parent="2" relname="adversative-concession">Although the experiment was carefully # designed,</segment>
109
+ <segment id="2" parent="4" relname="span">the results were inconclusive.</segment>
110
+ <segment id="3" parent="5" relname="span">We plan to repeat it tonight.</segment>
111
+ <group id="4" type="span" parent="3" relname="adversative-antithesis"/>
112
+ <group id="5" type="span"/>
113
+ </body>
114
+ </rst>
115
+ ```
116
+
117
+ ## Inference CLI
118
+
119
+ To identify a model on the command line, you may use a configuration file (`--config`), a PyTorch checkpoint (`--checkpoint`), or a HuggingFace Hub repository (`--hub-id`).
120
+
121
+ To provide input, you may specify an inline string (`--text`), a path to a raw text file or directory (`--text-file`, for parsers which support this), or an RS3/RS4 file or directory with gold EDUs already supplied (`--input`).
122
+
123
+ For `--text-file` and `--input`, results are written to `--output-dir` as `.rs4` files.
124
+
125
+ ```
126
+ # From the Hub, end-to-end on a directory of .txt files:
127
+ iudex dmrst predict \
128
+ --hub-id larc-iu/dmrst-gum-12.1.0 \
129
+ --text-file path/to/docs/ \
130
+ --output-dir out/ \
131
+ --device cuda
132
+
133
+ # From an explicit checkpoint:
134
+ iudex dmrst predict \
135
+ --checkpoint checkpoints/<run_id>/best_model.pt \
136
+ --text-file path/to/doc.txt \
137
+ --output-dir out/
138
+
139
+ # From a trained run's config, parsing pre-segmented RS3/RS4 with gold EDUs:
140
+ iudex topdown_biaffine predict \
141
+ --config configs/topdown_biaffine_rstdt.jsonnet \
142
+ --input data/rstdt/test \
143
+ --output-dir out/
144
+ ```
145
+
146
+ ## Available Models
147
+ All official IUDEX model releases are [tagged with `iudex` on the HuggingFace Hub](https://huggingface.co/models?other=iudex).
148
+
149
+ ## Training
150
+
151
+ To train a new top-down biaffine parser on RSTDT:
152
+
153
+ ```
154
+ iudex topdown_biaffine train configs/topdown_biaffine_rstdt.jsonnet
155
+ ```
156
+
157
+ Note that `configs/topdown_biaffine_rstdt.jsonnet` is a configuration.
158
+ You may either edit it directly or copy and modify it in a new location.
159
+
160
+ ### Grabbing Example Configurations
161
+
162
+ Model configurations required for training are not bundled with the package distributed via PyPI.
163
+
164
+ To get them you may visit [the associated directory](https://github.com/larc-iu/iudex/tree/master/configs) and download the configurations you're interested in manually.
165
+
166
+ If you want to grab all of them at once, you can use the command line like so:
167
+
168
+ **bash / zsh / macOS / Linux:**
169
+
170
+ ```bash
171
+ curl -fL https://github.com/larc-iu/iudex/archive/refs/heads/master.tar.gz \
172
+ | tar -xz --strip-components=1 --wildcards '*/configs'
173
+ ```
174
+
175
+ **Windows PowerShell:**
176
+
177
+ ```powershell
178
+ Invoke-WebRequest https://github.com/larc-iu/iudex/archive/refs/heads/master.zip -OutFile iudex.zip
179
+ Expand-Archive iudex.zip -DestinationPath .
180
+ Move-Item iudex-master/configs configs
181
+ Remove-Item -Recurse -Force iudex-master, iudex.zip
182
+ ```
183
+
184
+ Either leaves you with a local `configs/` directory you can edit and pass to `iudex … train configs/<name>.jsonnet`.
185
+
186
+ ### Configuration Hashes
187
+
188
+ Your configuration is used as the basis for a unique hash, which (by default) corresponds to a directory under `checkpoints/`.
189
+ This hash is used for several purposes.
190
+ For example, running the same config again resumes from the last epoch's checkpoint `last.pt` automatically if the run was interrupted.
191
+
192
+ To view all runs and their status, you may run the `runs list` subcommand:
193
+
194
+ ```
195
+ $ iudex runs list
196
+ Runs in checkpoints
197
+ ┏━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━┳━━━━━━━━━━━━━━━━━━┓
198
+ ┃ run_id ┃ run_name ┃ parser ┃ model_name ┃ train_dir ┃ best_val ┃ step ┃ modified ┃
199
+ ┡━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━╇━━━━━━━━━━━━━━━━━━┩
200
+ │ 245b1d774676 │ - │ dmrst │ xlm-roberta-base │ data/gum_12.1.0/train │ 0.3099 │ 1704 │ 2026-05-18 18:02 │
201
+ │ 41bc0fe1dd50 │ - │ topdown_biaffine │ SpanBERT/spanbert-base-cased │ data/rstdt/train │ 0.7576 │ 2149 │ 2026-05-18 13:51 │
202
+ │ 91525e48d63d │ - │ topdown_biaffine │ SpanBERT/spanbert-base-cased │ data/gum_12.1.0/train │ 0.6364 │ 1899 │ 2026-05-18 14:31 │
203
+ │ ad934ca992d4 │ - │ dmrst │ xlm-roberta-base │ data/rstdt/train │ 0.4665 │ 3090 │ 2026-05-18 16:46 │
204
+ └──────────────┴──────────┴──────────────────┴──────────────────────────────┴───────────────────────┴───────────┴──────┴──────────────────┘
205
+ ```
206
+
207
+ ### Monitoring with TensorBoard
208
+
209
+ Every run writes TensorBoard scalars (train loss, learning rate, gradient norm, and dev metrics) to `<run_dir>/tb/`. Point TensorBoard at your checkpoints directory to watch any run live or compare runs:
210
+
211
+ ```
212
+ tensorboard --logdir checkpoints/
213
+ ```
214
+
215
+ ### Pushing Models to HF Hub
216
+ You may host a trained model using each parser's `push` subcommand.
217
+ Each uploads `best_model.pt`, `config.json`, and an auto-generated `README.md` in a single commit:
218
+
219
+ ```
220
+ iudex topdown_biaffine push \
221
+ --config configs/topdown_biaffine_rstdt.jsonnet \
222
+ --repo-id larc-iu/topdown_biaffine-rstdt-coarse \
223
+ [--private] [--message "..."] [--token $HF_TOKEN]
224
+ ```
225
+
226
+ ## Citation
227
+
228
+ If you use IUDEX in your research, please cite it as:
229
+
230
+ > Gessler, Luke. 2026. *IUDEX: The Indiana University Discourse Exhibition.* https://github.com/larc-iu/iudex.
231
+
232
+ BibTeX:
233
+
234
+ ```bibtex
235
+ @misc{gessler-iudex-2026,
236
+ author = {Gessler, Luke},
237
+ title = {{IUDEX: The Indiana University Discourse Exhibition}},
238
+ year = {2026},
239
+ howpublished = {\url{https://github.com/larc-iu/iudex}},
240
+ }
241
+ ```
242
+
243
+ If you use one of the included parser re-implementations, please **also** cite the original paper (see each model's Hub card for the canonical reference).
@@ -0,0 +1,203 @@
1
+ # IUDEX
2
+
3
+ The **<u>I</u>ndiana <u>U</u>niversity <u>D</u>iscourse <u>Ex</u>hibition** (IUDEX) is a collection of parsers and other code related to discourse parsing.
4
+
5
+ ## Setup
6
+
7
+ For the latest release:
8
+
9
+ ```
10
+ pip install larc-iudex
11
+ ```
12
+
13
+ For the current state of `master`:
14
+
15
+ ```
16
+ pip install git+https://github.com/larc-iu/iudex
17
+ ```
18
+
19
+ Or for development:
20
+
21
+ ```
22
+ git clone https://github.com/larc-iu/iudex && cd iudex
23
+ pip install -e .
24
+ ```
25
+
26
+ Note that the command you will invoke is `iudex`, not `larc-iudex`.
27
+
28
+ ## Quick Start with Inference
29
+
30
+ Parse a sample document end-to-end with a pretrained DMRST model pulled from the HuggingFace Hub. From the command line:
31
+
32
+ ```bash
33
+ iudex dmrst predict \
34
+ --hub-id larc-iu/dmrst-gum-12.1.0 \
35
+ --text "Although the experiment was carefully designed, the results were inconclusive. We plan to repeat it tonight."
36
+ ```
37
+ This yields the parsed tree in `.rs3` format printed to `stdout`:
38
+ ```xml
39
+ <rst>
40
+ <relations><!-- ... --></relations>
41
+ <body>
42
+ <segment id="1" parent="2" relname="adversative-concession">Although the experiment was carefully # designed,</segment>
43
+ <segment id="2" parent="4" relname="span">the results were inconclusive.</segment>
44
+ <segment id="3" parent="5" relname="span">We plan to repeat it tonight.</segment>
45
+ <group id="4" type="span" parent="3" relname="adversative-antithesis"/>
46
+ <group id="5" type="span"/>
47
+ </body>
48
+ </rst>
49
+ ```
50
+
51
+ The same flow from Python:
52
+
53
+ ```python
54
+ from iudex.rst.parsers.dmrst.modeling_dmrst import DMRSTParser
55
+ parser = DMRSTParser.from_pretrained("larc-iu/dmrst-gum-12.1.0")
56
+ tree = parser.predict_from_text(
57
+ "Although the experiment was carefully designed, "
58
+ "the results were inconclusive. "
59
+ "We plan to repeat it tonight."
60
+ )
61
+ print(tree.to_rs4_string())
62
+ ```
63
+ Yields:
64
+ ```xml
65
+ <rst>
66
+ <relations><!-- ... --></relations>
67
+ <body>
68
+ <segment id="1" parent="2" relname="adversative-concession">Although the experiment was carefully # designed,</segment>
69
+ <segment id="2" parent="4" relname="span">the results were inconclusive.</segment>
70
+ <segment id="3" parent="5" relname="span">We plan to repeat it tonight.</segment>
71
+ <group id="4" type="span" parent="3" relname="adversative-antithesis"/>
72
+ <group id="5" type="span"/>
73
+ </body>
74
+ </rst>
75
+ ```
76
+
77
+ ## Inference CLI
78
+
79
+ To identify a model on the command line, you may use a configuration file (`--config`), a PyTorch checkpoint (`--checkpoint`), or a HuggingFace Hub repository (`--hub-id`).
80
+
81
+ To provide input, you may specify an inline string (`--text`), a path to a raw text file or directory (`--text-file`, for parsers which support this), or an RS3/RS4 file or directory with gold EDUs already supplied (`--input`).
82
+
83
+ For `--text-file` and `--input`, results are written to `--output-dir` as `.rs4` files.
84
+
85
+ ```
86
+ # From the Hub, end-to-end on a directory of .txt files:
87
+ iudex dmrst predict \
88
+ --hub-id larc-iu/dmrst-gum-12.1.0 \
89
+ --text-file path/to/docs/ \
90
+ --output-dir out/ \
91
+ --device cuda
92
+
93
+ # From an explicit checkpoint:
94
+ iudex dmrst predict \
95
+ --checkpoint checkpoints/<run_id>/best_model.pt \
96
+ --text-file path/to/doc.txt \
97
+ --output-dir out/
98
+
99
+ # From a trained run's config, parsing pre-segmented RS3/RS4 with gold EDUs:
100
+ iudex topdown_biaffine predict \
101
+ --config configs/topdown_biaffine_rstdt.jsonnet \
102
+ --input data/rstdt/test \
103
+ --output-dir out/
104
+ ```
105
+
106
+ ## Available Models
107
+ All official IUDEX model releases are [tagged with `iudex` on the HuggingFace Hub](https://huggingface.co/models?other=iudex).
108
+
109
+ ## Training
110
+
111
+ To train a new top-down biaffine parser on RSTDT:
112
+
113
+ ```
114
+ iudex topdown_biaffine train configs/topdown_biaffine_rstdt.jsonnet
115
+ ```
116
+
117
+ Note that `configs/topdown_biaffine_rstdt.jsonnet` is a configuration.
118
+ You may either edit it directly or copy and modify it in a new location.
119
+
120
+ ### Grabbing Example Configurations
121
+
122
+ Model configurations required for training are not bundled with the package distributed via PyPI.
123
+
124
+ To get them you may visit [the associated directory](https://github.com/larc-iu/iudex/tree/master/configs) and download the configurations you're interested in manually.
125
+
126
+ If you want to grab all of them at once, you can use the command line like so:
127
+
128
+ **bash / zsh / macOS / Linux:**
129
+
130
+ ```bash
131
+ curl -fL https://github.com/larc-iu/iudex/archive/refs/heads/master.tar.gz \
132
+ | tar -xz --strip-components=1 --wildcards '*/configs'
133
+ ```
134
+
135
+ **Windows PowerShell:**
136
+
137
+ ```powershell
138
+ Invoke-WebRequest https://github.com/larc-iu/iudex/archive/refs/heads/master.zip -OutFile iudex.zip
139
+ Expand-Archive iudex.zip -DestinationPath .
140
+ Move-Item iudex-master/configs configs
141
+ Remove-Item -Recurse -Force iudex-master, iudex.zip
142
+ ```
143
+
144
+ Either leaves you with a local `configs/` directory you can edit and pass to `iudex … train configs/<name>.jsonnet`.
145
+
146
+ ### Configuration Hashes
147
+
148
+ Your configuration is used as the basis for a unique hash, which (by default) corresponds to a directory under `checkpoints/`.
149
+ This hash is used for several purposes.
150
+ For example, running the same config again resumes from the last epoch's checkpoint `last.pt` automatically if the run was interrupted.
151
+
152
+ To view all runs and their status, you may run the `runs list` subcommand:
153
+
154
+ ```
155
+ $ iudex runs list
156
+ Runs in checkpoints
157
+ ┏━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━┳━━━━━━━━━━━━━━━━━━┓
158
+ ┃ run_id ┃ run_name ┃ parser ┃ model_name ┃ train_dir ┃ best_val ┃ step ┃ modified ┃
159
+ ┡━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━╇━━━━━━━━━━━━━━━━━━┩
160
+ │ 245b1d774676 │ - │ dmrst │ xlm-roberta-base │ data/gum_12.1.0/train │ 0.3099 │ 1704 │ 2026-05-18 18:02 │
161
+ │ 41bc0fe1dd50 │ - │ topdown_biaffine │ SpanBERT/spanbert-base-cased │ data/rstdt/train │ 0.7576 │ 2149 │ 2026-05-18 13:51 │
162
+ │ 91525e48d63d │ - │ topdown_biaffine │ SpanBERT/spanbert-base-cased │ data/gum_12.1.0/train │ 0.6364 │ 1899 │ 2026-05-18 14:31 │
163
+ │ ad934ca992d4 │ - │ dmrst │ xlm-roberta-base │ data/rstdt/train │ 0.4665 │ 3090 │ 2026-05-18 16:46 │
164
+ └──────────────┴──────────┴──────────────────┴──────────────────────────────┴───────────────────────┴───────────┴──────┴──────────────────┘
165
+ ```
166
+
167
+ ### Monitoring with TensorBoard
168
+
169
+ Every run writes TensorBoard scalars (train loss, learning rate, gradient norm, and dev metrics) to `<run_dir>/tb/`. Point TensorBoard at your checkpoints directory to watch any run live or compare runs:
170
+
171
+ ```
172
+ tensorboard --logdir checkpoints/
173
+ ```
174
+
175
+ ### Pushing Models to HF Hub
176
+ You may host a trained model using each parser's `push` subcommand.
177
+ Each uploads `best_model.pt`, `config.json`, and an auto-generated `README.md` in a single commit:
178
+
179
+ ```
180
+ iudex topdown_biaffine push \
181
+ --config configs/topdown_biaffine_rstdt.jsonnet \
182
+ --repo-id larc-iu/topdown_biaffine-rstdt-coarse \
183
+ [--private] [--message "..."] [--token $HF_TOKEN]
184
+ ```
185
+
186
+ ## Citation
187
+
188
+ If you use IUDEX in your research, please cite it as:
189
+
190
+ > Gessler, Luke. 2026. *IUDEX: The Indiana University Discourse Exhibition.* https://github.com/larc-iu/iudex.
191
+
192
+ BibTeX:
193
+
194
+ ```bibtex
195
+ @misc{gessler-iudex-2026,
196
+ author = {Gessler, Luke},
197
+ title = {{IUDEX: The Indiana University Discourse Exhibition}},
198
+ year = {2026},
199
+ howpublished = {\url{https://github.com/larc-iu/iudex}},
200
+ }
201
+ ```
202
+
203
+ If you use one of the included parser re-implementations, please **also** cite the original paper (see each model's Hub card for the canonical reference).
@@ -0,0 +1,22 @@
1
+ """This module declares the project's CLI surface that lives above any one
2
+ framework:
3
+
4
+ - `FRAMEWORKS`: dotted paths of framework modules (e.g. `iudex.rst`).
5
+ Each framework module exposes `PARSERS`, `PARSER_SCOPED_COMMANDS`,
6
+ and `GLOBAL_COMMANDS`. The dispatcher (`iudex/__main__.py`) imports
7
+ each and merges the three.
8
+ - `GLOBAL_COMMANDS`: `{cmd: module_path}` for project-level commands
9
+ that aren't owned by any single framework (e.g. `runs`, which walks
10
+ every framework's parser registry to tag rows by parser kind).
11
+
12
+ To add a sibling framework, append its dotted path to `FRAMEWORKS` and
13
+ give its `__init__.py` the three required attributes.
14
+ """
15
+
16
+ __version__ = "0.1.0"
17
+
18
+ FRAMEWORKS: list[str] = ["iudex.rst"]
19
+
20
+ GLOBAL_COMMANDS: dict[str, str] = {
21
+ "runs": "iudex.runs",
22
+ }
@@ -0,0 +1,96 @@
1
+ """iudex CLI dispatcher.
2
+
3
+ `iudex <parser> <cmd>` or `iudex <cmd>` for globals. Routes by merging the
4
+ registries declared by each framework in `iudex.FRAMEWORKS`. See
5
+ `iudex/__init__.py` for the framework contract.
6
+ """
7
+
8
+ import importlib
9
+ import sys
10
+
11
+ import iudex
12
+
13
+
14
+ def _merge_frameworks() -> tuple[dict, dict, dict]:
15
+ """Merge each framework's three registry dicts atop project-level
16
+ `iudex.GLOBAL_COMMANDS`. Aborts on a name collision."""
17
+ parsers: dict = {}
18
+ parser_scoped: dict = {}
19
+ global_cmds: dict = dict(iudex.GLOBAL_COMMANDS)
20
+ for fw_path in iudex.FRAMEWORKS:
21
+ fw = importlib.import_module(fw_path)
22
+ _merge_no_collide(parsers, fw.PARSERS, fw_path, "parser name")
23
+ _merge_no_collide(parser_scoped, fw.PARSER_SCOPED_COMMANDS, fw_path, "parser-scoped command")
24
+ _merge_no_collide(global_cmds, fw.GLOBAL_COMMANDS, fw_path, "global command")
25
+ return parsers, parser_scoped, global_cmds
26
+
27
+
28
+ def _merge_no_collide(dst: dict, src: dict, fw_path: str, kind: str) -> None:
29
+ for k, v in src.items():
30
+ if k in dst and dst[k] is not v:
31
+ sys.stderr.write(
32
+ f"iudex: {kind} {k!r} declared by both an earlier framework and {fw_path!r}. Rename one of them.\n"
33
+ )
34
+ sys.exit(2)
35
+ dst[k] = v
36
+
37
+
38
+ PARSERS, PARSER_SCOPED_COMMANDS, GLOBAL_COMMANDS = _merge_frameworks()
39
+
40
+
41
+ def main():
42
+ if len(sys.argv) > 1 and sys.argv[1] in ("-h", "--help"):
43
+ print(__doc__.strip())
44
+ print(f"\nKnown parsers: {', '.join(sorted(PARSERS))}")
45
+ print(f"Global commands: {', '.join(sorted(GLOBAL_COMMANDS))}")
46
+ sys.exit(0)
47
+
48
+ if len(sys.argv) < 2:
49
+ sys.stderr.write(__doc__.strip() + "\n\n")
50
+ sys.stderr.write(f"Known parsers: {', '.join(sorted(PARSERS))}\n")
51
+ sys.stderr.write(f"Global commands: {', '.join(sorted(GLOBAL_COMMANDS))}\n")
52
+ sys.exit(2)
53
+
54
+ head = sys.argv[1]
55
+
56
+ if head in GLOBAL_COMMANDS:
57
+ module = importlib.import_module(GLOBAL_COMMANDS[head])
58
+ sys.argv = [GLOBAL_COMMANDS[head]] + sys.argv[2:]
59
+ module.main()
60
+ return
61
+
62
+ if len(sys.argv) < 3:
63
+ sys.stderr.write(__doc__.strip() + "\n\n")
64
+ sys.stderr.write(f"Known parsers: {', '.join(sorted(PARSERS))}\n")
65
+ sys.exit(2)
66
+
67
+ parser_name = head
68
+ command = sys.argv[2]
69
+
70
+ if parser_name not in PARSERS:
71
+ sys.stderr.write(f"Unknown parser: {parser_name!r}\n")
72
+ sys.stderr.write(f"Known parsers: {', '.join(sorted(PARSERS))}\n")
73
+ sys.stderr.write(f"Global commands: {', '.join(sorted(GLOBAL_COMMANDS))}\n")
74
+ sys.exit(2)
75
+
76
+ if command in PARSER_SCOPED_COMMANDS:
77
+ shared_path = PARSER_SCOPED_COMMANDS[command]
78
+ module = importlib.import_module(shared_path)
79
+ sys.argv = [f"iudex {parser_name} {command}"] + sys.argv[3:]
80
+ module.main(parser_kind=parser_name)
81
+ return
82
+
83
+ module_path = f"{PARSERS[parser_name].package}.{command}_{parser_name}"
84
+ try:
85
+ module = importlib.import_module(module_path)
86
+ except ImportError as e:
87
+ sys.stderr.write(f"No such command {command!r} for parser {parser_name!r}\n")
88
+ sys.stderr.write(f" (tried to import {module_path}: {e})\n")
89
+ sys.exit(2)
90
+
91
+ sys.argv = [module_path] + sys.argv[3:]
92
+ module.main()
93
+
94
+
95
+ if __name__ == "__main__":
96
+ main()
File without changes
@@ -0,0 +1,65 @@
1
+ """Rich-based logging and console helpers."""
2
+
3
+ import logging
4
+ import os
5
+
6
+ from rich.console import Console
7
+ from rich.logging import RichHandler
8
+ from rich.theme import Theme
9
+
10
+ theme = Theme(
11
+ {
12
+ "info": "cyan",
13
+ "warning": "yellow",
14
+ "error": "bold red",
15
+ "metric": "bold green",
16
+ "metric.name": "dim",
17
+ "epoch": "bold magenta",
18
+ "step": "dim cyan",
19
+ "lr": "dim yellow",
20
+ "loss": "bold orange1",
21
+ "gpu": "bold green",
22
+ "path": "underline blue",
23
+ }
24
+ )
25
+
26
+ console = Console(theme=theme)
27
+
28
+
29
+ def success(msg: str) -> None:
30
+ console.print(f"[bold green]{msg}[/bold green]")
31
+
32
+
33
+ def warn(msg: str) -> None:
34
+ console.print(f"[bold yellow]{msg}[/bold yellow]")
35
+
36
+
37
+ def dim(msg: str) -> None:
38
+ console.print(f"[dim]{msg}[/dim]")
39
+
40
+
41
+ def rule(title: str) -> None:
42
+ console.rule(f"[bold magenta]{title}[/bold magenta]")
43
+
44
+
45
+ def wrote(path: str) -> None:
46
+ """Announce a file the program just wrote, with its absolute path."""
47
+ console.print(f"[dim]Wrote[/dim] [path]{os.path.abspath(path)}[/path]")
48
+
49
+
50
+ def setup_logging(level: int = logging.INFO) -> None:
51
+ logging.basicConfig(
52
+ level=level,
53
+ format="%(message)s",
54
+ datefmt="[%X]",
55
+ handlers=[
56
+ RichHandler(
57
+ console=console,
58
+ rich_tracebacks=True,
59
+ tracebacks_show_locals=True,
60
+ show_path=False,
61
+ markup=True,
62
+ )
63
+ ],
64
+ force=True,
65
+ )