archival-structures 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- archival_structures-0.1.0/LICENSE +21 -0
- archival_structures-0.1.0/PKG-INFO +154 -0
- archival_structures-0.1.0/README.md +114 -0
- archival_structures-0.1.0/archival_structures/__init__.py +5 -0
- archival_structures-0.1.0/archival_structures/analysis/allen_interval.py +362 -0
- archival_structures-0.1.0/archival_structures/analysis/grid_analysis.py +254 -0
- archival_structures-0.1.0/archival_structures/analysis/line_clustering.py +104 -0
- archival_structures-0.1.0/archival_structures/analysis/neighbourhood_analysis.py +121 -0
- archival_structures-0.1.0/archival_structures/analysis/opening_detection.py +396 -0
- archival_structures-0.1.0/archival_structures/analysis/page.py +106 -0
- archival_structures-0.1.0/archival_structures/analysis/page_layout_clustering.py +52 -0
- archival_structures-0.1.0/archival_structures/analysis/region_calculus.py +81 -0
- archival_structures-0.1.0/archival_structures/analysis/sequence_patterns.py +230 -0
- archival_structures-0.1.0/archival_structures/analysis/stream_analysis.py +126 -0
- archival_structures-0.1.0/archival_structures/analysis/text_analysis.py +226 -0
- archival_structures-0.1.0/archival_structures/analysis/token_analysis.py +81 -0
- archival_structures-0.1.0/archival_structures/clustering/colour_clustering.py +1199 -0
- archival_structures-0.1.0/archival_structures/clustering/colour_quantisation.py +74 -0
- archival_structures-0.1.0/archival_structures/clustering/image_clustering.py +295 -0
- archival_structures-0.1.0/archival_structures/clustering/peaks.py +281 -0
- archival_structures-0.1.0/archival_structures/clustering/thumbnail_color_clustering.py +0 -0
- archival_structures-0.1.0/archival_structures/datasets/__init__.py +4 -0
- archival_structures-0.1.0/archival_structures/datasets/annotations.py +166 -0
- archival_structures-0.1.0/archival_structures/datasets/images_transcriptions.py +109 -0
- archival_structures-0.1.0/archival_structures/image/__init__.py +4 -0
- archival_structures-0.1.0/archival_structures/image/crop_borders.py +428 -0
- archival_structures-0.1.0/archival_structures/image/image_annotation.py +207 -0
- archival_structures-0.1.0/archival_structures/image/image_base.py +134 -0
- archival_structures-0.1.0/archival_structures/image/image_drawing.py +448 -0
- archival_structures-0.1.0/archival_structures/image/image_processing.py +475 -0
- archival_structures-0.1.0/archival_structures/image/pagexml_bridge.py +83 -0
- archival_structures-0.1.0/archival_structures/model/image.py +230 -0
- archival_structures-0.1.0/archival_structures/model/inventory_number.py +91 -0
- archival_structures-0.1.0/archival_structures/parsers/__init__.py +1 -0
- archival_structures-0.1.0/archival_structures/parsers/ead_parser.py +699 -0
- archival_structures-0.1.0/archival_structures/parsers/ead_start_end_year.py +91 -0
- archival_structures-0.1.0/archival_structures/parsers/mets_parser.py +86 -0
- archival_structures-0.1.0/archival_structures/parsers/read.py +188 -0
- archival_structures-0.1.0/archival_structures/stats/token_stats.py +4 -0
- archival_structures-0.1.0/archival_structures/utils/file.py +15 -0
- archival_structures-0.1.0/archival_structures/utils/image_utils.py +103 -0
- archival_structures-0.1.0/archival_structures/utils/wordlist.py +31 -0
- archival_structures-0.1.0/pyproject.toml +57 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Marijn Koolen
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: archival-structures
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A Python package for analysing and restructuring the output of Automatic Text Recognition (ATR) pipelines.
|
|
5
|
+
License: MIT
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Keywords: archives,pagexml,atr,ocr,digital-humanities,document-analysis
|
|
8
|
+
Author: Marijn Koolen
|
|
9
|
+
Author-email: marijn.koolen@gmail.com
|
|
10
|
+
Requires-Python: >=3.11,<3.15
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering :: Image Recognition
|
|
19
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
20
|
+
Requires-Dist: fuzzy-search (>=2.5.0,<4.0.0)
|
|
21
|
+
Requires-Dist: hdbscan (>=0.8.44)
|
|
22
|
+
Requires-Dist: ipycanvas (>=0.13.0)
|
|
23
|
+
Requires-Dist: ipywidgets (>=8.1.0)
|
|
24
|
+
Requires-Dist: lxml (>=5.3.0,<7.0.0)
|
|
25
|
+
Requires-Dist: numpy (>=2.3.5,<3.0.0)
|
|
26
|
+
Requires-Dist: opencv-python (>=4.13.0.92)
|
|
27
|
+
Requires-Dist: orjson (>=3.11.7)
|
|
28
|
+
Requires-Dist: pagexml-tools (>=0.8.0,<1.0.0)
|
|
29
|
+
Requires-Dist: pandas (>=2.3.3,<3.0.0)
|
|
30
|
+
Requires-Dist: scikit-image (>=0.26.0)
|
|
31
|
+
Requires-Dist: scikit-learn (>=1.9.0)
|
|
32
|
+
Requires-Dist: torch (>=2.12.0)
|
|
33
|
+
Requires-Dist: transformers (>=5.11.0)
|
|
34
|
+
Requires-Dist: umap-learn (>=0.5.12)
|
|
35
|
+
Project-URL: Homepage, https://github.com/Data-Scopes/archival-structures
|
|
36
|
+
Project-URL: Issues, https://github.com/Data-Scopes/archival-structures/issues
|
|
37
|
+
Project-URL: Repository, https://github.com/Data-Scopes/archival-structures
|
|
38
|
+
Description-Content-Type: text/markdown
|
|
39
|
+
|
|
40
|
+
# archival-structures
|
|
41
|
+
|
|
42
|
+
Tools for analysing PageXML/ATR transcriptions and scan images of archival documents:
|
|
43
|
+
detecting and splitting two-page book openings, clustering text lines and page layouts,
|
|
44
|
+
mining cross-page document-element sequences, ink-colour and missing-transcription detection,
|
|
45
|
+
and parsing EAD/METS archival finding-aid metadata.
|
|
46
|
+
|
|
47
|
+
Full documentation (including the per-module API reference) lives in [`docs/`](docs/) and is
|
|
48
|
+
built with Sphinx; see [Documentation](#documentation) below.
|
|
49
|
+
|
|
50
|
+
## Techniques and tasks
|
|
51
|
+
|
|
52
|
+
Archival images and transcriptions are organised as
|
|
53
|
+
`<institute_id>/<archive_id>/<inventory_num_id>/<scan>`. The core idea behind this package is
|
|
54
|
+
that one inventory number's worth of scans is a structured, ordered corpus, not a set of
|
|
55
|
+
independent images -- so the analysis is built up in layers:
|
|
56
|
+
|
|
57
|
+
1. **Opening detection and splitting** (`archival_structures.analysis.opening_detection`) --
|
|
58
|
+
decide whether a scan is a two-page spread, split it into independent verso/recto pages, and
|
|
59
|
+
classify a whole inventory number as a *book of openings* versus a *mixed* folder/booklet.
|
|
60
|
+
2. **Page-layout clustering** (`archival_structures.analysis.page_layout_clustering`) -- cluster
|
|
61
|
+
whole pages by the spatial arrangement of their text lines, via a grid-pattern TF-IDF
|
|
62
|
+
fingerprint.
|
|
63
|
+
3. **Line clustering** (`archival_structures.analysis.line_clustering`) -- cluster individual
|
|
64
|
+
text lines by indentation/width/height into a vocabulary of recurring line types (body text,
|
|
65
|
+
closing lines, marginalia, ...).
|
|
66
|
+
4. **Sequence-pattern mining** (`archival_structures.analysis.sequence_patterns`) -- order lines
|
|
67
|
+
into a corpus-wide reading sequence and segment it into document elements, including elements
|
|
68
|
+
that span a page break.
|
|
69
|
+
|
|
70
|
+
Tasks 2 and 3 both depend on splitting first (task 1) -- clustering whole two-page scans
|
|
71
|
+
conflates the left and right page's geometry into one coordinate frame.
|
|
72
|
+
|
|
73
|
+
Alongside the text-analysis pipeline:
|
|
74
|
+
|
|
75
|
+
- **Ink colour, multi-colour text, and missing transcriptions**
|
|
76
|
+
(`archival_structures.clustering.colour_clustering`) -- robust ink/paper separation via
|
|
77
|
+
multiotsu + connected-component shape (resistant to small artefacts like a sticker or stain),
|
|
78
|
+
screening pages for more than one ink colour via LAB chroma spread, and flagging untranscribed
|
|
79
|
+
page regions whose pixels look like genuine ink rather than blank paper.
|
|
80
|
+
- **Coordinate-space bridging** (`archival_structures.model.image`,
|
|
81
|
+
`archival_structures.image`) -- converting between a scan's native pixel coordinates, a
|
|
82
|
+
thumbnail's, and a canvas rendering of a selection, via an affine `Transform`; converting
|
|
83
|
+
between PageXML `Coords` and this package's own `Box` type; ipywidgets-based interactive
|
|
84
|
+
region drawing/tagging.
|
|
85
|
+
- **Ground-truth annotation** (`archival_structures.datasets.annotations`) -- a JSON schema for
|
|
86
|
+
labelling scans/lines/cross-page elements, plus an ipywidgets notebook app for producing it.
|
|
87
|
+
- **EAD/METS parsing** (`archival_structures.parsers`) -- a separate concern from the
|
|
88
|
+
PageXML/image pipeline: parsing the archival finding-aid metadata (series/subseries/file
|
|
89
|
+
structure, page manifests) that describes an archive's holdings.
|
|
90
|
+
|
|
91
|
+
See [`docs/findings.md`](docs/findings.md) for the concrete, validated-against-real-data lessons
|
|
92
|
+
learned while building this -- several of the choices above (e.g. splitting before clustering,
|
|
93
|
+
chroma spread over luminosity-class counting for multi-colour detection) turned out to matter a
|
|
94
|
+
lot more than they first appeared to.
|
|
95
|
+
|
|
96
|
+
## Demo notebooks
|
|
97
|
+
|
|
98
|
+
All in [`notebooks/demo/`](notebooks/demo/):
|
|
99
|
+
|
|
100
|
+
- [`annotate-scans.ipynb`](notebooks/demo/annotate-scans.ipynb) -- ipywidgets ground-truth
|
|
101
|
+
annotation app.
|
|
102
|
+
- [`inventory-structure-demo.ipynb`](notebooks/demo/inventory-structure-demo.ipynb) --
|
|
103
|
+
classifying a whole inventory number as a book of openings vs a mixed folder.
|
|
104
|
+
- [`opening-detection-demo.ipynb`](notebooks/demo/opening-detection-demo.ipynb) -- per-scan
|
|
105
|
+
opening detection and splitting.
|
|
106
|
+
- [`line-clustering-demo.ipynb`](notebooks/demo/line-clustering-demo.ipynb) and
|
|
107
|
+
[`line-clustering-table-vs-deeds-demo.ipynb`](notebooks/demo/line-clustering-table-vs-deeds-demo.ipynb)
|
|
108
|
+
-- clustering text lines by indentation/width, and comparing that across a table-like register
|
|
109
|
+
versus notary deeds.
|
|
110
|
+
- [`page-layout-clustering-demo.ipynb`](notebooks/demo/page-layout-clustering-demo.ipynb) and
|
|
111
|
+
[`page-layout-clustering-table-vs-deeds-demo.ipynb`](notebooks/demo/page-layout-clustering-table-vs-deeds-demo.ipynb)
|
|
112
|
+
-- clustering pages by text-line layout, and the same table-vs-deeds comparison.
|
|
113
|
+
- [`pagexml-image-region-linking.ipynb`](notebooks/demo/pagexml-image-region-linking.ipynb) --
|
|
114
|
+
drawing PageXML regions on a thumbnail, and converting a manually-drawn selection back into a
|
|
115
|
+
new PageXML region.
|
|
116
|
+
- [`pagexml-image-multicolour-explorer.ipynb`](notebooks/demo/pagexml-image-multicolour-explorer.ipynb)
|
|
117
|
+
-- screening a sample of scans for multi-colour text and missing-transcription candidates.
|
|
118
|
+
- [`sequence-patterns-demo.ipynb`](notebooks/demo/sequence-patterns-demo.ipynb) -- mining
|
|
119
|
+
recurring n-gram patterns and cross-page document elements, comparing the table register
|
|
120
|
+
against the notary deeds.
|
|
121
|
+
|
|
122
|
+
### Demo data
|
|
123
|
+
|
|
124
|
+
The notebooks above need real PageXML/thumbnail data (~341MB across 7 inventory numbers) that
|
|
125
|
+
isn't committed to this repo -- only the package code is. Download `demo-data.zip` from the
|
|
126
|
+
[latest release](https://github.com/Data-Scopes/archival-structures/releases) and extract it at
|
|
127
|
+
the repository root:
|
|
128
|
+
|
|
129
|
+
```bash
|
|
130
|
+
unzip demo-data.zip -d .
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
This recreates `data/PageXML/`, `data/thumbs/`, and `data/annotations/` with exactly the
|
|
134
|
+
inventory numbers the demo notebooks reference, so they run unchanged once extracted.
|
|
135
|
+
|
|
136
|
+
## Installation
|
|
137
|
+
|
|
138
|
+
```bash
|
|
139
|
+
poetry install
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
Requires Python >=3.11,<3.15 -- `torch`'s `triton` dependency caps out at Python <3.15, so the
|
|
143
|
+
project's declared Python range matches that rather than the more typical `<4.0`.
|
|
144
|
+
|
|
145
|
+
## Documentation
|
|
146
|
+
|
|
147
|
+
Built with Sphinx; requires the optional `docs` dependency group:
|
|
148
|
+
|
|
149
|
+
```bash
|
|
150
|
+
poetry install --with docs
|
|
151
|
+
cd docs
|
|
152
|
+
make html
|
|
153
|
+
```
|
|
154
|
+
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
# archival-structures
|
|
2
|
+
|
|
3
|
+
Tools for analysing PageXML/ATR transcriptions and scan images of archival documents:
|
|
4
|
+
detecting and splitting two-page book openings, clustering text lines and page layouts,
|
|
5
|
+
mining cross-page document-element sequences, ink-colour and missing-transcription detection,
|
|
6
|
+
and parsing EAD/METS archival finding-aid metadata.
|
|
7
|
+
|
|
8
|
+
Full documentation (including the per-module API reference) lives in [`docs/`](docs/) and is
|
|
9
|
+
built with Sphinx; see [Documentation](#documentation) below.
|
|
10
|
+
|
|
11
|
+
## Techniques and tasks
|
|
12
|
+
|
|
13
|
+
Archival images and transcriptions are organised as
|
|
14
|
+
`<institute_id>/<archive_id>/<inventory_num_id>/<scan>`. The core idea behind this package is
|
|
15
|
+
that one inventory number's worth of scans is a structured, ordered corpus, not a set of
|
|
16
|
+
independent images -- so the analysis is built up in layers:
|
|
17
|
+
|
|
18
|
+
1. **Opening detection and splitting** (`archival_structures.analysis.opening_detection`) --
|
|
19
|
+
decide whether a scan is a two-page spread, split it into independent verso/recto pages, and
|
|
20
|
+
classify a whole inventory number as a *book of openings* versus a *mixed* folder/booklet.
|
|
21
|
+
2. **Page-layout clustering** (`archival_structures.analysis.page_layout_clustering`) -- cluster
|
|
22
|
+
whole pages by the spatial arrangement of their text lines, via a grid-pattern TF-IDF
|
|
23
|
+
fingerprint.
|
|
24
|
+
3. **Line clustering** (`archival_structures.analysis.line_clustering`) -- cluster individual
|
|
25
|
+
text lines by indentation/width/height into a vocabulary of recurring line types (body text,
|
|
26
|
+
closing lines, marginalia, ...).
|
|
27
|
+
4. **Sequence-pattern mining** (`archival_structures.analysis.sequence_patterns`) -- order lines
|
|
28
|
+
into a corpus-wide reading sequence and segment it into document elements, including elements
|
|
29
|
+
that span a page break.
|
|
30
|
+
|
|
31
|
+
Tasks 2 and 3 both depend on splitting first (task 1) -- clustering whole two-page scans
|
|
32
|
+
conflates the left and right page's geometry into one coordinate frame.
|
|
33
|
+
|
|
34
|
+
Alongside the text-analysis pipeline:
|
|
35
|
+
|
|
36
|
+
- **Ink colour, multi-colour text, and missing transcriptions**
|
|
37
|
+
(`archival_structures.clustering.colour_clustering`) -- robust ink/paper separation via
|
|
38
|
+
multiotsu + connected-component shape (resistant to small artefacts like a sticker or stain),
|
|
39
|
+
screening pages for more than one ink colour via LAB chroma spread, and flagging untranscribed
|
|
40
|
+
page regions whose pixels look like genuine ink rather than blank paper.
|
|
41
|
+
- **Coordinate-space bridging** (`archival_structures.model.image`,
|
|
42
|
+
`archival_structures.image`) -- converting between a scan's native pixel coordinates, a
|
|
43
|
+
thumbnail's, and a canvas rendering of a selection, via an affine `Transform`; converting
|
|
44
|
+
between PageXML `Coords` and this package's own `Box` type; ipywidgets-based interactive
|
|
45
|
+
region drawing/tagging.
|
|
46
|
+
- **Ground-truth annotation** (`archival_structures.datasets.annotations`) -- a JSON schema for
|
|
47
|
+
labelling scans/lines/cross-page elements, plus an ipywidgets notebook app for producing it.
|
|
48
|
+
- **EAD/METS parsing** (`archival_structures.parsers`) -- a separate concern from the
|
|
49
|
+
PageXML/image pipeline: parsing the archival finding-aid metadata (series/subseries/file
|
|
50
|
+
structure, page manifests) that describes an archive's holdings.
|
|
51
|
+
|
|
52
|
+
See [`docs/findings.md`](docs/findings.md) for the concrete, validated-against-real-data lessons
|
|
53
|
+
learned while building this -- several of the choices above (e.g. splitting before clustering,
|
|
54
|
+
chroma spread over luminosity-class counting for multi-colour detection) turned out to matter a
|
|
55
|
+
lot more than they first appeared to.
|
|
56
|
+
|
|
57
|
+
## Demo notebooks
|
|
58
|
+
|
|
59
|
+
All in [`notebooks/demo/`](notebooks/demo/):
|
|
60
|
+
|
|
61
|
+
- [`annotate-scans.ipynb`](notebooks/demo/annotate-scans.ipynb) -- ipywidgets ground-truth
|
|
62
|
+
annotation app.
|
|
63
|
+
- [`inventory-structure-demo.ipynb`](notebooks/demo/inventory-structure-demo.ipynb) --
|
|
64
|
+
classifying a whole inventory number as a book of openings vs a mixed folder.
|
|
65
|
+
- [`opening-detection-demo.ipynb`](notebooks/demo/opening-detection-demo.ipynb) -- per-scan
|
|
66
|
+
opening detection and splitting.
|
|
67
|
+
- [`line-clustering-demo.ipynb`](notebooks/demo/line-clustering-demo.ipynb) and
|
|
68
|
+
[`line-clustering-table-vs-deeds-demo.ipynb`](notebooks/demo/line-clustering-table-vs-deeds-demo.ipynb)
|
|
69
|
+
-- clustering text lines by indentation/width, and comparing that across a table-like register
|
|
70
|
+
versus notary deeds.
|
|
71
|
+
- [`page-layout-clustering-demo.ipynb`](notebooks/demo/page-layout-clustering-demo.ipynb) and
|
|
72
|
+
[`page-layout-clustering-table-vs-deeds-demo.ipynb`](notebooks/demo/page-layout-clustering-table-vs-deeds-demo.ipynb)
|
|
73
|
+
-- clustering pages by text-line layout, and the same table-vs-deeds comparison.
|
|
74
|
+
- [`pagexml-image-region-linking.ipynb`](notebooks/demo/pagexml-image-region-linking.ipynb) --
|
|
75
|
+
drawing PageXML regions on a thumbnail, and converting a manually-drawn selection back into a
|
|
76
|
+
new PageXML region.
|
|
77
|
+
- [`pagexml-image-multicolour-explorer.ipynb`](notebooks/demo/pagexml-image-multicolour-explorer.ipynb)
|
|
78
|
+
-- screening a sample of scans for multi-colour text and missing-transcription candidates.
|
|
79
|
+
- [`sequence-patterns-demo.ipynb`](notebooks/demo/sequence-patterns-demo.ipynb) -- mining
|
|
80
|
+
recurring n-gram patterns and cross-page document elements, comparing the table register
|
|
81
|
+
against the notary deeds.
|
|
82
|
+
|
|
83
|
+
### Demo data
|
|
84
|
+
|
|
85
|
+
The notebooks above need real PageXML/thumbnail data (~341MB across 7 inventory numbers) that
|
|
86
|
+
isn't committed to this repo -- only the package code is. Download `demo-data.zip` from the
|
|
87
|
+
[latest release](https://github.com/Data-Scopes/archival-structures/releases) and extract it at
|
|
88
|
+
the repository root:
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
unzip demo-data.zip -d .
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
This recreates `data/PageXML/`, `data/thumbs/`, and `data/annotations/` with exactly the
|
|
95
|
+
inventory numbers the demo notebooks reference, so they run unchanged once extracted.
|
|
96
|
+
|
|
97
|
+
## Installation
|
|
98
|
+
|
|
99
|
+
```bash
|
|
100
|
+
poetry install
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
Requires Python >=3.11,<3.15 -- `torch`'s `triton` dependency caps out at Python <3.15, so the
|
|
104
|
+
project's declared Python range matches that rather than the more typical `<4.0`.
|
|
105
|
+
|
|
106
|
+
## Documentation
|
|
107
|
+
|
|
108
|
+
Built with Sphinx; requires the optional `docs` dependency group:
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
poetry install --with docs
|
|
112
|
+
cd docs
|
|
113
|
+
make html
|
|
114
|
+
```
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
"""Tools for analysing PageXML/ATR transcriptions and scan images of archival documents:
|
|
2
|
+
detecting and splitting two-page book openings, clustering text lines and page layouts,
|
|
3
|
+
mining cross-page document-element sequences, ink-colour and missing-transcription detection,
|
|
4
|
+
and parsing EAD/METS archival finding-aid metadata.
|
|
5
|
+
"""
|
|
@@ -0,0 +1,362 @@
|
|
|
1
|
+
"""Allen Interval Algebra relations between PageXML regions (lines/regions treated as 1D
|
|
2
|
+
horizontal intervals).
|
|
3
|
+
|
|
4
|
+
`get_allen_interval` classifies a pair of regions into one of the 13 base Allen relations
|
|
5
|
+
(e.g. ``before``, ``meets``, ``overlaps``, ``equal``). `get_extended_allen_interval` further
|
|
6
|
+
splits the asymmetric ``overlaps``/``contains``/``starts``/``finishes`` relations by how much
|
|
7
|
+
of each region is covered (``m`` = most, ``l`` = least), giving the finer-grained
|
|
8
|
+
`EXTENDED_ALLEN_INTERVALS` vocabulary -- useful for distinguishing e.g. a small marginal
|
|
9
|
+
annotation overlapping a body-text line from two lines of comparable width overlapping evenly.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from collections import Counter
|
|
13
|
+
|
|
14
|
+
import pagexml.model.physical_document_model as pdm
|
|
15
|
+
from pagexml.helper.pagexml_helper import horizontal_group_lines
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
ALLEN_INTERVALS = {
|
|
19
|
+
'after', 'meets', 'o',
|
|
20
|
+
'fi', 'di', 's', 'equal', 'si', 'd', 'f',
|
|
21
|
+
'oi', 'met by', 'before'
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
EXTENDED_ALLEN_INTERVALS = {
|
|
25
|
+
'after', 'meets',
|
|
26
|
+
'mom', 'mol', 'lom', 'lol',
|
|
27
|
+
'm', 'mfi', 'lfi',
|
|
28
|
+
'ldi', 'cdi', 'rdi',
|
|
29
|
+
'ms', 'ls', 'equal', 'lsi', 'msi',
|
|
30
|
+
'ld', 'cd', 'rd', 'lf', 'mf',
|
|
31
|
+
'momi', 'moli', 'lomi', 'loli',
|
|
32
|
+
'met by', 'before'
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def get_doc_allen_interval_profile(doc: pdm.PageXMLRegion, use_extended: bool = True):
|
|
37
|
+
"""Frequency count of Allen interval relations between consecutive horizontally-grouped
|
|
38
|
+
lines of `doc` (each line compared against the previous and next group), giving a profile
|
|
39
|
+
of how `doc`'s lines relate to their immediate neighbours. Uses
|
|
40
|
+
`get_extended_allen_interval` if `use_extended`, else the base `get_allen_interval`."""
|
|
41
|
+
interval_func = get_extended_allen_interval if use_extended else get_allen_interval
|
|
42
|
+
hor_group_lines = horizontal_group_lines(doc.get_lines())
|
|
43
|
+
prev_group = None
|
|
44
|
+
ai_freq = Counter()
|
|
45
|
+
for ci, curr_group in enumerate(hor_group_lines):
|
|
46
|
+
if prev_group is not None:
|
|
47
|
+
ai_freq.update([interval_func(pl, cl) for cl in curr_group for pl in prev_group])
|
|
48
|
+
if ci < len(hor_group_lines) - 1:
|
|
49
|
+
next_group = hor_group_lines[ci+1]
|
|
50
|
+
ai_freq.update([interval_func(cl, nl) for cl in curr_group for nl in next_group])
|
|
51
|
+
prev_group = curr_group
|
|
52
|
+
return ai_freq
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def get_normalised_diff(x: int, y: int, max_diff: int = 0) -> int:
|
|
56
|
+
"""`x - y`, snapped to 0 if the difference is within `max_diff` (treats near-identical
|
|
57
|
+
boundaries as exactly aligned)."""
|
|
58
|
+
diff = x - y
|
|
59
|
+
return 0 if abs(diff) <= max_diff else diff
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def determine_overlap_relation(region1: pdm.PageXMLDoc, region2: pdm.PageXMLDoc) -> tuple[str | None, str | None]:
|
|
63
|
+
"""For each region, whether the horizontal overlap with the other covers at least half
|
|
64
|
+
of its own width (``'m'``, most) or less (``'l'``, least). Returns `(None, None)` if the
|
|
65
|
+
regions don't overlap horizontally at all."""
|
|
66
|
+
overlap = pdm.get_horizontal_overlap(region1, region2)
|
|
67
|
+
if overlap == 0:
|
|
68
|
+
return None, None
|
|
69
|
+
relation1 = 'm' if overlap / region1.coords.width >= 0.5 else 'l'
|
|
70
|
+
relation2 = 'm' if overlap / region2.coords.width >= 0.5 else 'l'
|
|
71
|
+
return relation1, relation2
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def get_normalised_region_diffs(region1: pdm.PageXMLDoc, region2: pdm.PageXMLDoc) -> tuple[int, int, int, int]:
|
|
75
|
+
"""Left/right boundary differences between `region1` and `region2` (left-vs-left,
|
|
76
|
+
left-vs-right, right-vs-left, right-vs-right), using baselines if both regions have one,
|
|
77
|
+
else bounding boxes. These four values are what `get_allen_interval`/
|
|
78
|
+
`get_extended_allen_interval` branch on to determine the interval relation."""
|
|
79
|
+
if pdm.has_baseline(region1) and pdm.has_baseline(region2):
|
|
80
|
+
# only use baseline when both have them, otherwise, use bounding boxes
|
|
81
|
+
coords1 = region1.baseline
|
|
82
|
+
coords2 = region2.baseline
|
|
83
|
+
else:
|
|
84
|
+
coords1, coords2 = region1.coords, region2.coords
|
|
85
|
+
l1l2 = get_normalised_diff(coords1.left, coords2.left)
|
|
86
|
+
l1r2 = get_normalised_diff(coords1.left, coords2.right)
|
|
87
|
+
r1l2 = get_normalised_diff(coords1.right, coords2.left)
|
|
88
|
+
r1r2 = get_normalised_diff(coords1.right, coords2.right)
|
|
89
|
+
return l1l2, l1r2, r1l2, r1r2
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def get_allen_interval(region1: pdm.PageXMLDoc, region2: pdm.PageXMLDoc,
|
|
93
|
+
debug: int = 0):
|
|
94
|
+
"""Determine the (Extended) Allen Interval.
|
|
95
|
+
|
|
96
|
+
Use `max_diff` to work with thick boundaries.
|
|
97
|
+
|
|
98
|
+
:param region1: a PageXML document with spatial coordinates
|
|
99
|
+
:param region2: a PageXML document with spatial coordinates
|
|
100
|
+
:param debug: print debugging info
|
|
101
|
+
"""
|
|
102
|
+
l1l2, l1r2, r1l2, r1r2 = get_normalised_region_diffs(region1, region2)
|
|
103
|
+
overlap1, overlap2 = determine_overlap_relation(region1, region2)
|
|
104
|
+
if debug > 0:
|
|
105
|
+
print(f"allen_interval.get_extended_allen_interval\n\tl1: left: {pdm.coords_as_span(region1.coords)}")
|
|
106
|
+
print(f"allen_interval.get_extended_allen_interval\n\tl2: left: {pdm.coords_as_span(region2.coords)}")
|
|
107
|
+
print(f"allen_interval.get_extended_allen_interval\n\tl1l2: {l1l2} r1r2: {r1r2} "
|
|
108
|
+
f"r1l2: {r1l2}\n\toverlap1: {overlap1} overlap2: {overlap2}")
|
|
109
|
+
if l1l2 < 0:
|
|
110
|
+
if r1r2 < 0:
|
|
111
|
+
if r1l2 < 0:
|
|
112
|
+
# --
|
|
113
|
+
# --
|
|
114
|
+
return 'after'
|
|
115
|
+
elif r1l2 == 0:
|
|
116
|
+
# ---
|
|
117
|
+
# ---
|
|
118
|
+
return 'meets'
|
|
119
|
+
else:
|
|
120
|
+
# ---
|
|
121
|
+
# -----
|
|
122
|
+
return 'o'
|
|
123
|
+
elif r1r2 == 0:
|
|
124
|
+
# ------
|
|
125
|
+
# ----
|
|
126
|
+
return "fi"
|
|
127
|
+
else:
|
|
128
|
+
# ------
|
|
129
|
+
# --
|
|
130
|
+
return 'di'
|
|
131
|
+
elif l1l2 == 0:
|
|
132
|
+
if r1r2 < 0:
|
|
133
|
+
# ----
|
|
134
|
+
# ------
|
|
135
|
+
return 's'
|
|
136
|
+
elif r1r2 == 0:
|
|
137
|
+
# no shift, complete overlap (equal)
|
|
138
|
+
# ------
|
|
139
|
+
# ------
|
|
140
|
+
return 'equal'
|
|
141
|
+
else:
|
|
142
|
+
# r1r2 > 0
|
|
143
|
+
# ------
|
|
144
|
+
# ----
|
|
145
|
+
return 'si'
|
|
146
|
+
else:
|
|
147
|
+
# l1l2 > 0
|
|
148
|
+
if r1r2 < 0:
|
|
149
|
+
# --
|
|
150
|
+
# ------
|
|
151
|
+
return 'd'
|
|
152
|
+
elif r1r2 == 0:
|
|
153
|
+
# ----
|
|
154
|
+
# ------
|
|
155
|
+
return 'f'
|
|
156
|
+
else:
|
|
157
|
+
# r1r2 > 0
|
|
158
|
+
if l1r2 < 0:
|
|
159
|
+
# -----
|
|
160
|
+
# ---
|
|
161
|
+
return 'oi'
|
|
162
|
+
elif l1r2 == 0:
|
|
163
|
+
# ---
|
|
164
|
+
# ---
|
|
165
|
+
return 'met by'
|
|
166
|
+
else:
|
|
167
|
+
# --
|
|
168
|
+
# --
|
|
169
|
+
return 'before'
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def get_extended_allen_interval(region1: pdm.PageXMLDoc, region2: pdm.PageXMLDoc,
|
|
173
|
+
debug: int = 0):
|
|
174
|
+
"""Determine the (Extended) Allen Interval.
|
|
175
|
+
|
|
176
|
+
Use `max_diff` to work with thick boundaries.
|
|
177
|
+
|
|
178
|
+
:param region1: a PageXML document with spatial coordinates
|
|
179
|
+
:param region2: a PageXML document with spatial coordinates
|
|
180
|
+
:param debug: print debugging info
|
|
181
|
+
"""
|
|
182
|
+
l1l2, l1r2, r1l2, r1r2 = get_normalised_region_diffs(region1, region2)
|
|
183
|
+
overlap1, overlap2 = determine_overlap_relation(region1, region2)
|
|
184
|
+
if debug > 0:
|
|
185
|
+
print(f"allen_interval.get_extended_allen_interval\n\tl1: left: {pdm.coords_as_span(region1.coords)}")
|
|
186
|
+
print(f"allen_interval.get_extended_allen_interval\n\tl2: left: {pdm.coords_as_span(region2.coords)}")
|
|
187
|
+
print(f"allen_interval.get_extended_allen_interval\n\tl1l2: {l1l2} r1r2: {r1r2} "
|
|
188
|
+
f"r1l2: {r1l2}\n\toverlap1: {overlap1} overlap2: {overlap2}")
|
|
189
|
+
if l1l2 < 0:
|
|
190
|
+
if r1r2 < 0:
|
|
191
|
+
if r1l2 < 0:
|
|
192
|
+
# --
|
|
193
|
+
# --
|
|
194
|
+
return 'after'
|
|
195
|
+
elif r1l2 == 0:
|
|
196
|
+
# ---
|
|
197
|
+
# ---
|
|
198
|
+
return 'meets'
|
|
199
|
+
else:
|
|
200
|
+
if overlap1 == 'l' and overlap2 == 'm':
|
|
201
|
+
# right-shift, shrink, most part 2 overlapped by least part 1 (lom)
|
|
202
|
+
# -----
|
|
203
|
+
# ---
|
|
204
|
+
return 'lom'
|
|
205
|
+
elif overlap1 == 'm' and overlap2 == 'l':
|
|
206
|
+
# right-shift, stretch, most part 1 overlaps least part 2 (mol)
|
|
207
|
+
# ---
|
|
208
|
+
# -----
|
|
209
|
+
return 'mol'
|
|
210
|
+
elif overlap1 == 'm' and overlap2 == 'm':
|
|
211
|
+
# right-shift, most part overlaps most part (mom)
|
|
212
|
+
# -----
|
|
213
|
+
# -----
|
|
214
|
+
return 'mom'
|
|
215
|
+
elif overlap1 == 'l' and overlap2 == 'l':
|
|
216
|
+
# right-shift, least part overlaps least part (lol)
|
|
217
|
+
# ----
|
|
218
|
+
# ---
|
|
219
|
+
return 'lol'
|
|
220
|
+
else:
|
|
221
|
+
raise ValueError(f"unexpected situation - l1l2: {l1l2} r1r2: {r1r2} "
|
|
222
|
+
f"r1l2: {r1l2} overlap1: {overlap1} overlap2: {overlap2}")
|
|
223
|
+
elif r1r2 == 0:
|
|
224
|
+
if overlap1 == 'm':
|
|
225
|
+
# right-shift, finished by, most part overlap (mfi)
|
|
226
|
+
# ------
|
|
227
|
+
# ----
|
|
228
|
+
return 'mfi'
|
|
229
|
+
elif overlap1 == 'l':
|
|
230
|
+
# right-shift, finished by, least part overlap (lfi)
|
|
231
|
+
# ------
|
|
232
|
+
# --
|
|
233
|
+
return "lfi"
|
|
234
|
+
else:
|
|
235
|
+
raise ValueError(f"unexpected situation - l1l2: {l1l2} r1r2: {r1r2} "
|
|
236
|
+
f"r1l2: {r1l2} l1r2: {l1r2} "
|
|
237
|
+
f"overlap1: {overlap1} overlap2: {overlap2}")
|
|
238
|
+
else:
|
|
239
|
+
diff = get_normalised_diff(abs(l1l2), abs(r1r2))
|
|
240
|
+
if diff < 0:
|
|
241
|
+
# right-shift, contained by, right-aligned
|
|
242
|
+
# ------
|
|
243
|
+
# --
|
|
244
|
+
return 'ldi'
|
|
245
|
+
elif diff == 0:
|
|
246
|
+
# no shift, contained by, center-aligned
|
|
247
|
+
# ------
|
|
248
|
+
# ----
|
|
249
|
+
return 'cdi'
|
|
250
|
+
else:
|
|
251
|
+
# left-shift, contained by, left-aligned
|
|
252
|
+
# ------
|
|
253
|
+
# --
|
|
254
|
+
return 'rdi'
|
|
255
|
+
elif l1l2 == 0:
|
|
256
|
+
if r1r2 < 0:
|
|
257
|
+
if overlap2 == 'm':
|
|
258
|
+
# right-shift, starts, overlaps with most part (ms)
|
|
259
|
+
# ----
|
|
260
|
+
# ------
|
|
261
|
+
return 'ms'
|
|
262
|
+
elif overlap2 == 'l':
|
|
263
|
+
# right-shift, starts, overlaps with least part (ls)
|
|
264
|
+
# --
|
|
265
|
+
# ------
|
|
266
|
+
return 'ls'
|
|
267
|
+
else:
|
|
268
|
+
raise ValueError(f"unexpected situation - l1l2: {l1l2} r1r2: {r1r2} "
|
|
269
|
+
f"r1l2: {r1l2} l1r2: {l1r2} "
|
|
270
|
+
f"overlap1: {overlap1} overlap2: {overlap2}")
|
|
271
|
+
elif r1r2 == 0:
|
|
272
|
+
# no shift, complete overlap (equal)
|
|
273
|
+
# ------
|
|
274
|
+
# ------
|
|
275
|
+
return 'equal'
|
|
276
|
+
else:
|
|
277
|
+
# r1r2 > 0
|
|
278
|
+
if overlap1 == 'm':
|
|
279
|
+
# left-shift, starts, most part overlaps
|
|
280
|
+
# ------
|
|
281
|
+
# ----
|
|
282
|
+
return 'msi'
|
|
283
|
+
elif overlap1 == 'l':
|
|
284
|
+
# left-shift, starts, least part overlaps
|
|
285
|
+
# ------
|
|
286
|
+
# --
|
|
287
|
+
return 'lsi'
|
|
288
|
+
else:
|
|
289
|
+
raise ValueError(f"unexpected situation - l1l2: {l1l2} r1r2: {r1r2} "
|
|
290
|
+
f"r1l2: {r1l2} l1r2: {l1r2} "
|
|
291
|
+
f"overlap1: {overlap1} overlap2: {overlap2}")
|
|
292
|
+
else:
|
|
293
|
+
# l1l2 > 0
|
|
294
|
+
if r1r2 < 0:
|
|
295
|
+
diff = get_normalised_diff(abs(l1l2), abs(r1r2))
|
|
296
|
+
if diff < 0:
|
|
297
|
+
# right-shift, contains, left-aligned
|
|
298
|
+
# --
|
|
299
|
+
# ------
|
|
300
|
+
return 'ld'
|
|
301
|
+
elif diff == 0:
|
|
302
|
+
# no shift, contains, center-aligned
|
|
303
|
+
# ----
|
|
304
|
+
# ------
|
|
305
|
+
return 'cd'
|
|
306
|
+
else:
|
|
307
|
+
# left-shift, contains, right-aligned
|
|
308
|
+
# --
|
|
309
|
+
# ------
|
|
310
|
+
return 'rd'
|
|
311
|
+
elif r1r2 == 0:
|
|
312
|
+
if overlap2 == 'm':
|
|
313
|
+
# left-shift, finishes, most part overlap (mfi)
|
|
314
|
+
# ----
|
|
315
|
+
# ------
|
|
316
|
+
return 'mf'
|
|
317
|
+
elif overlap2 == 'l':
|
|
318
|
+
# left-shift, finishes, least part overlap (lfi)
|
|
319
|
+
# --
|
|
320
|
+
# ------
|
|
321
|
+
return 'lf'
|
|
322
|
+
else:
|
|
323
|
+
raise ValueError(f"unexpected situation - l1l2: {l1l2} r1r2: {r1r2} "
|
|
324
|
+
f"r1l2: {r1l2} l1r2: {l1r2} "
|
|
325
|
+
f"overlap1: {overlap1} overlap2: {overlap2}")
|
|
326
|
+
else:
|
|
327
|
+
# r1r2 > 0
|
|
328
|
+
if l1r2 < 0:
|
|
329
|
+
if overlap1 == 'l' and overlap2 == 'm':
|
|
330
|
+
# left-shift, shrink, least part 2 overlapped by most part 1 (lomi)
|
|
331
|
+
# -----
|
|
332
|
+
# ---
|
|
333
|
+
return 'lomi'
|
|
334
|
+
elif overlap1 == 'm' and overlap2 == 'l':
|
|
335
|
+
# left-shift, stretch, least part 2 overlaps most part 1 (moli)
|
|
336
|
+
# ---
|
|
337
|
+
# -----
|
|
338
|
+
return 'moli'
|
|
339
|
+
elif overlap1 == 'm' and overlap2 == 'm':
|
|
340
|
+
# left-shift, most part overlaps most part (momi)
|
|
341
|
+
# -----
|
|
342
|
+
# -----
|
|
343
|
+
return 'momi'
|
|
344
|
+
elif overlap1 == 'l' and overlap2 == 'l':
|
|
345
|
+
# left-shift, least part overlaps least part (loli)
|
|
346
|
+
# ---
|
|
347
|
+
# ----
|
|
348
|
+
return 'loli'
|
|
349
|
+
else:
|
|
350
|
+
print(f"region 1: {region1.id} {region1.coords.box_string}")
|
|
351
|
+
print(f"region 2: {region2.id} {region2.coords.box_string}")
|
|
352
|
+
raise ValueError(f"unexpected situation - l1l2: {l1l2} r1r2: {r1r2} "
|
|
353
|
+
f"r1l2: {r1l2} l1r2: {l1r2} "
|
|
354
|
+
f"overlap1: {overlap1} overlap2: {overlap2}")
|
|
355
|
+
elif l1r2 == 0:
|
|
356
|
+
# ---
|
|
357
|
+
# ---
|
|
358
|
+
return 'met by'
|
|
359
|
+
else:
|
|
360
|
+
# --
|
|
361
|
+
# --
|
|
362
|
+
return 'before'
|