archival-structures 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. archival_structures-0.1.0/LICENSE +21 -0
  2. archival_structures-0.1.0/PKG-INFO +154 -0
  3. archival_structures-0.1.0/README.md +114 -0
  4. archival_structures-0.1.0/archival_structures/__init__.py +5 -0
  5. archival_structures-0.1.0/archival_structures/analysis/allen_interval.py +362 -0
  6. archival_structures-0.1.0/archival_structures/analysis/grid_analysis.py +254 -0
  7. archival_structures-0.1.0/archival_structures/analysis/line_clustering.py +104 -0
  8. archival_structures-0.1.0/archival_structures/analysis/neighbourhood_analysis.py +121 -0
  9. archival_structures-0.1.0/archival_structures/analysis/opening_detection.py +396 -0
  10. archival_structures-0.1.0/archival_structures/analysis/page.py +106 -0
  11. archival_structures-0.1.0/archival_structures/analysis/page_layout_clustering.py +52 -0
  12. archival_structures-0.1.0/archival_structures/analysis/region_calculus.py +81 -0
  13. archival_structures-0.1.0/archival_structures/analysis/sequence_patterns.py +230 -0
  14. archival_structures-0.1.0/archival_structures/analysis/stream_analysis.py +126 -0
  15. archival_structures-0.1.0/archival_structures/analysis/text_analysis.py +226 -0
  16. archival_structures-0.1.0/archival_structures/analysis/token_analysis.py +81 -0
  17. archival_structures-0.1.0/archival_structures/clustering/colour_clustering.py +1199 -0
  18. archival_structures-0.1.0/archival_structures/clustering/colour_quantisation.py +74 -0
  19. archival_structures-0.1.0/archival_structures/clustering/image_clustering.py +295 -0
  20. archival_structures-0.1.0/archival_structures/clustering/peaks.py +281 -0
  21. archival_structures-0.1.0/archival_structures/clustering/thumbnail_color_clustering.py +0 -0
  22. archival_structures-0.1.0/archival_structures/datasets/__init__.py +4 -0
  23. archival_structures-0.1.0/archival_structures/datasets/annotations.py +166 -0
  24. archival_structures-0.1.0/archival_structures/datasets/images_transcriptions.py +109 -0
  25. archival_structures-0.1.0/archival_structures/image/__init__.py +4 -0
  26. archival_structures-0.1.0/archival_structures/image/crop_borders.py +428 -0
  27. archival_structures-0.1.0/archival_structures/image/image_annotation.py +207 -0
  28. archival_structures-0.1.0/archival_structures/image/image_base.py +134 -0
  29. archival_structures-0.1.0/archival_structures/image/image_drawing.py +448 -0
  30. archival_structures-0.1.0/archival_structures/image/image_processing.py +475 -0
  31. archival_structures-0.1.0/archival_structures/image/pagexml_bridge.py +83 -0
  32. archival_structures-0.1.0/archival_structures/model/image.py +230 -0
  33. archival_structures-0.1.0/archival_structures/model/inventory_number.py +91 -0
  34. archival_structures-0.1.0/archival_structures/parsers/__init__.py +1 -0
  35. archival_structures-0.1.0/archival_structures/parsers/ead_parser.py +699 -0
  36. archival_structures-0.1.0/archival_structures/parsers/ead_start_end_year.py +91 -0
  37. archival_structures-0.1.0/archival_structures/parsers/mets_parser.py +86 -0
  38. archival_structures-0.1.0/archival_structures/parsers/read.py +188 -0
  39. archival_structures-0.1.0/archival_structures/stats/token_stats.py +4 -0
  40. archival_structures-0.1.0/archival_structures/utils/file.py +15 -0
  41. archival_structures-0.1.0/archival_structures/utils/image_utils.py +103 -0
  42. archival_structures-0.1.0/archival_structures/utils/wordlist.py +31 -0
  43. archival_structures-0.1.0/pyproject.toml +57 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Marijn Koolen
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,154 @@
1
+ Metadata-Version: 2.4
2
+ Name: archival-structures
3
+ Version: 0.1.0
4
+ Summary: A Python package for analysing and restructuring the output of Automatic Text Recognition (ATR) pipelines.
5
+ License: MIT
6
+ License-File: LICENSE
7
+ Keywords: archives,pagexml,atr,ocr,digital-humanities,document-analysis
8
+ Author: Marijn Koolen
9
+ Author-email: marijn.koolen@gmail.com
10
+ Requires-Python: >=3.11,<3.15
11
+ Classifier: Development Status :: 3 - Alpha
12
+ Classifier: Intended Audience :: Science/Research
13
+ Classifier: License :: OSI Approved :: MIT License
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.11
16
+ Classifier: Programming Language :: Python :: 3.12
17
+ Classifier: Programming Language :: Python :: 3.13
18
+ Classifier: Topic :: Scientific/Engineering :: Image Recognition
19
+ Classifier: Topic :: Text Processing :: Linguistic
20
+ Requires-Dist: fuzzy-search (>=2.5.0,<4.0.0)
21
+ Requires-Dist: hdbscan (>=0.8.44)
22
+ Requires-Dist: ipycanvas (>=0.13.0)
23
+ Requires-Dist: ipywidgets (>=8.1.0)
24
+ Requires-Dist: lxml (>=5.3.0,<7.0.0)
25
+ Requires-Dist: numpy (>=2.3.5,<3.0.0)
26
+ Requires-Dist: opencv-python (>=4.13.0.92)
27
+ Requires-Dist: orjson (>=3.11.7)
28
+ Requires-Dist: pagexml-tools (>=0.8.0,<1.0.0)
29
+ Requires-Dist: pandas (>=2.3.3,<3.0.0)
30
+ Requires-Dist: scikit-image (>=0.26.0)
31
+ Requires-Dist: scikit-learn (>=1.9.0)
32
+ Requires-Dist: torch (>=2.12.0)
33
+ Requires-Dist: transformers (>=5.11.0)
34
+ Requires-Dist: umap-learn (>=0.5.12)
35
+ Project-URL: Homepage, https://github.com/Data-Scopes/archival-structures
36
+ Project-URL: Issues, https://github.com/Data-Scopes/archival-structures/issues
37
+ Project-URL: Repository, https://github.com/Data-Scopes/archival-structures
38
+ Description-Content-Type: text/markdown
39
+
40
+ # archival-structures
41
+
42
+ Tools for analysing PageXML/ATR transcriptions and scan images of archival documents:
43
+ detecting and splitting two-page book openings, clustering text lines and page layouts,
44
+ mining cross-page document-element sequences, ink-colour and missing-transcription detection,
45
+ and parsing EAD/METS archival finding-aid metadata.
46
+
47
+ Full documentation (including the per-module API reference) lives in [`docs/`](docs/) and is
48
+ built with Sphinx; see [Documentation](#documentation) below.
49
+
50
+ ## Techniques and tasks
51
+
52
+ Archival images and transcriptions are organised as
53
+ `<institute_id>/<archive_id>/<inventory_num_id>/<scan>`. The core idea behind this package is
54
+ that one inventory number's worth of scans is a structured, ordered corpus, not a set of
55
+ independent images -- so the analysis is built up in layers:
56
+
57
+ 1. **Opening detection and splitting** (`archival_structures.analysis.opening_detection`) --
58
+ decide whether a scan is a two-page spread, split it into independent verso/recto pages, and
59
+ classify a whole inventory number as a *book of openings* versus a *mixed* folder/booklet.
60
+ 2. **Page-layout clustering** (`archival_structures.analysis.page_layout_clustering`) -- cluster
61
+ whole pages by the spatial arrangement of their text lines, via a grid-pattern TF-IDF
62
+ fingerprint.
63
+ 3. **Line clustering** (`archival_structures.analysis.line_clustering`) -- cluster individual
64
+ text lines by indentation/width/height into a vocabulary of recurring line types (body text,
65
+ closing lines, marginalia, ...).
66
+ 4. **Sequence-pattern mining** (`archival_structures.analysis.sequence_patterns`) -- order lines
67
+ into a corpus-wide reading sequence and segment it into document elements, including elements
68
+ that span a page break.
69
+
70
+ Tasks 2 and 3 both depend on splitting first (task 1) -- clustering whole two-page scans
71
+ conflates the left and right page's geometry into one coordinate frame.
72
+
73
+ Alongside the text-analysis pipeline:
74
+
75
+ - **Ink colour, multi-colour text, and missing transcriptions**
76
+ (`archival_structures.clustering.colour_clustering`) -- robust ink/paper separation via
77
+ multiotsu + connected-component shape (resistant to small artefacts like a sticker or stain),
78
+ screening pages for more than one ink colour via LAB chroma spread, and flagging untranscribed
79
+ page regions whose pixels look like genuine ink rather than blank paper.
80
+ - **Coordinate-space bridging** (`archival_structures.model.image`,
81
+ `archival_structures.image`) -- converting between a scan's native pixel coordinates, a
82
+ thumbnail's, and a canvas rendering of a selection, via an affine `Transform`; converting
83
+ between PageXML `Coords` and this package's own `Box` type; ipywidgets-based interactive
84
+ region drawing/tagging.
85
+ - **Ground-truth annotation** (`archival_structures.datasets.annotations`) -- a JSON schema for
86
+ labelling scans/lines/cross-page elements, plus an ipywidgets notebook app for producing it.
87
+ - **EAD/METS parsing** (`archival_structures.parsers`) -- a separate concern from the
88
+ PageXML/image pipeline: parsing the archival finding-aid metadata (series/subseries/file
89
+ structure, page manifests) that describes an archive's holdings.
90
+
91
+ See [`docs/findings.md`](docs/findings.md) for the concrete, validated-against-real-data lessons
92
+ learned while building this -- several of the choices above (e.g. splitting before clustering,
93
+ chroma spread over luminosity-class counting for multi-colour detection) turned out to matter a
94
+ lot more than they first appeared to.
95
+
96
+ ## Demo notebooks
97
+
98
+ All in [`notebooks/demo/`](notebooks/demo/):
99
+
100
+ - [`annotate-scans.ipynb`](notebooks/demo/annotate-scans.ipynb) -- ipywidgets ground-truth
101
+ annotation app.
102
+ - [`inventory-structure-demo.ipynb`](notebooks/demo/inventory-structure-demo.ipynb) --
103
+ classifying a whole inventory number as a book of openings vs a mixed folder.
104
+ - [`opening-detection-demo.ipynb`](notebooks/demo/opening-detection-demo.ipynb) -- per-scan
105
+ opening detection and splitting.
106
+ - [`line-clustering-demo.ipynb`](notebooks/demo/line-clustering-demo.ipynb) and
107
+ [`line-clustering-table-vs-deeds-demo.ipynb`](notebooks/demo/line-clustering-table-vs-deeds-demo.ipynb)
108
+ -- clustering text lines by indentation/width, and comparing that across a table-like register
109
+ versus notary deeds.
110
+ - [`page-layout-clustering-demo.ipynb`](notebooks/demo/page-layout-clustering-demo.ipynb) and
111
+ [`page-layout-clustering-table-vs-deeds-demo.ipynb`](notebooks/demo/page-layout-clustering-table-vs-deeds-demo.ipynb)
112
+ -- clustering pages by text-line layout, and the same table-vs-deeds comparison.
113
+ - [`pagexml-image-region-linking.ipynb`](notebooks/demo/pagexml-image-region-linking.ipynb) --
114
+ drawing PageXML regions on a thumbnail, and converting a manually-drawn selection back into a
115
+ new PageXML region.
116
+ - [`pagexml-image-multicolour-explorer.ipynb`](notebooks/demo/pagexml-image-multicolour-explorer.ipynb)
117
+ -- screening a sample of scans for multi-colour text and missing-transcription candidates.
118
+ - [`sequence-patterns-demo.ipynb`](notebooks/demo/sequence-patterns-demo.ipynb) -- mining
119
+ recurring n-gram patterns and cross-page document elements, comparing the table register
120
+ against the notary deeds.
121
+
122
+ ### Demo data
123
+
124
+ The notebooks above need real PageXML/thumbnail data (~341MB across 7 inventory numbers) that
125
+ isn't committed to this repo -- only the package code is. Download `demo-data.zip` from the
126
+ [latest release](https://github.com/Data-Scopes/archival-structures/releases) and extract it at
127
+ the repository root:
128
+
129
+ ```bash
130
+ unzip demo-data.zip -d .
131
+ ```
132
+
133
+ This recreates `data/PageXML/`, `data/thumbs/`, and `data/annotations/` with exactly the
134
+ inventory numbers the demo notebooks reference, so they run unchanged once extracted.
135
+
136
+ ## Installation
137
+
138
+ ```bash
139
+ poetry install
140
+ ```
141
+
142
+ Requires Python >=3.11,<3.15 -- `torch`'s `triton` dependency caps out at Python <3.15, so the
143
+ project's declared Python range matches that rather than the more typical `<4.0`.
144
+
145
+ ## Documentation
146
+
147
+ Built with Sphinx; requires the optional `docs` dependency group:
148
+
149
+ ```bash
150
+ poetry install --with docs
151
+ cd docs
152
+ make html
153
+ ```
154
+
@@ -0,0 +1,114 @@
1
+ # archival-structures
2
+
3
+ Tools for analysing PageXML/ATR transcriptions and scan images of archival documents:
4
+ detecting and splitting two-page book openings, clustering text lines and page layouts,
5
+ mining cross-page document-element sequences, ink-colour and missing-transcription detection,
6
+ and parsing EAD/METS archival finding-aid metadata.
7
+
8
+ Full documentation (including the per-module API reference) lives in [`docs/`](docs/) and is
9
+ built with Sphinx; see [Documentation](#documentation) below.
10
+
11
+ ## Techniques and tasks
12
+
13
+ Archival images and transcriptions are organised as
14
+ `<institute_id>/<archive_id>/<inventory_num_id>/<scan>`. The core idea behind this package is
15
+ that one inventory number's worth of scans is a structured, ordered corpus, not a set of
16
+ independent images -- so the analysis is built up in layers:
17
+
18
+ 1. **Opening detection and splitting** (`archival_structures.analysis.opening_detection`) --
19
+ decide whether a scan is a two-page spread, split it into independent verso/recto pages, and
20
+ classify a whole inventory number as a *book of openings* versus a *mixed* folder/booklet.
21
+ 2. **Page-layout clustering** (`archival_structures.analysis.page_layout_clustering`) -- cluster
22
+ whole pages by the spatial arrangement of their text lines, via a grid-pattern TF-IDF
23
+ fingerprint.
24
+ 3. **Line clustering** (`archival_structures.analysis.line_clustering`) -- cluster individual
25
+ text lines by indentation/width/height into a vocabulary of recurring line types (body text,
26
+ closing lines, marginalia, ...).
27
+ 4. **Sequence-pattern mining** (`archival_structures.analysis.sequence_patterns`) -- order lines
28
+ into a corpus-wide reading sequence and segment it into document elements, including elements
29
+ that span a page break.
30
+
31
+ Tasks 2 and 3 both depend on splitting first (task 1) -- clustering whole two-page scans
32
+ conflates the left and right page's geometry into one coordinate frame.
33
+
34
+ Alongside the text-analysis pipeline:
35
+
36
+ - **Ink colour, multi-colour text, and missing transcriptions**
37
+ (`archival_structures.clustering.colour_clustering`) -- robust ink/paper separation via
38
+ multiotsu + connected-component shape (resistant to small artefacts like a sticker or stain),
39
+ screening pages for more than one ink colour via LAB chroma spread, and flagging untranscribed
40
+ page regions whose pixels look like genuine ink rather than blank paper.
41
+ - **Coordinate-space bridging** (`archival_structures.model.image`,
42
+ `archival_structures.image`) -- converting between a scan's native pixel coordinates, a
43
+ thumbnail's, and a canvas rendering of a selection, via an affine `Transform`; converting
44
+ between PageXML `Coords` and this package's own `Box` type; ipywidgets-based interactive
45
+ region drawing/tagging.
46
+ - **Ground-truth annotation** (`archival_structures.datasets.annotations`) -- a JSON schema for
47
+ labelling scans/lines/cross-page elements, plus an ipywidgets notebook app for producing it.
48
+ - **EAD/METS parsing** (`archival_structures.parsers`) -- a separate concern from the
49
+ PageXML/image pipeline: parsing the archival finding-aid metadata (series/subseries/file
50
+ structure, page manifests) that describes an archive's holdings.
51
+
52
+ See [`docs/findings.md`](docs/findings.md) for the concrete, validated-against-real-data lessons
53
+ learned while building this -- several of the choices above (e.g. splitting before clustering,
54
+ chroma spread over luminosity-class counting for multi-colour detection) turned out to matter a
55
+ lot more than they first appeared to.
56
+
57
+ ## Demo notebooks
58
+
59
+ All in [`notebooks/demo/`](notebooks/demo/):
60
+
61
+ - [`annotate-scans.ipynb`](notebooks/demo/annotate-scans.ipynb) -- ipywidgets ground-truth
62
+ annotation app.
63
+ - [`inventory-structure-demo.ipynb`](notebooks/demo/inventory-structure-demo.ipynb) --
64
+ classifying a whole inventory number as a book of openings vs a mixed folder.
65
+ - [`opening-detection-demo.ipynb`](notebooks/demo/opening-detection-demo.ipynb) -- per-scan
66
+ opening detection and splitting.
67
+ - [`line-clustering-demo.ipynb`](notebooks/demo/line-clustering-demo.ipynb) and
68
+ [`line-clustering-table-vs-deeds-demo.ipynb`](notebooks/demo/line-clustering-table-vs-deeds-demo.ipynb)
69
+ -- clustering text lines by indentation/width, and comparing that across a table-like register
70
+ versus notary deeds.
71
+ - [`page-layout-clustering-demo.ipynb`](notebooks/demo/page-layout-clustering-demo.ipynb) and
72
+ [`page-layout-clustering-table-vs-deeds-demo.ipynb`](notebooks/demo/page-layout-clustering-table-vs-deeds-demo.ipynb)
73
+ -- clustering pages by text-line layout, and the same table-vs-deeds comparison.
74
+ - [`pagexml-image-region-linking.ipynb`](notebooks/demo/pagexml-image-region-linking.ipynb) --
75
+ drawing PageXML regions on a thumbnail, and converting a manually-drawn selection back into a
76
+ new PageXML region.
77
+ - [`pagexml-image-multicolour-explorer.ipynb`](notebooks/demo/pagexml-image-multicolour-explorer.ipynb)
78
+ -- screening a sample of scans for multi-colour text and missing-transcription candidates.
79
+ - [`sequence-patterns-demo.ipynb`](notebooks/demo/sequence-patterns-demo.ipynb) -- mining
80
+ recurring n-gram patterns and cross-page document elements, comparing the table register
81
+ against the notary deeds.
82
+
83
+ ### Demo data
84
+
85
+ The notebooks above need real PageXML/thumbnail data (~341MB across 7 inventory numbers) that
86
+ isn't committed to this repo -- only the package code is. Download `demo-data.zip` from the
87
+ [latest release](https://github.com/Data-Scopes/archival-structures/releases) and extract it at
88
+ the repository root:
89
+
90
+ ```bash
91
+ unzip demo-data.zip -d .
92
+ ```
93
+
94
+ This recreates `data/PageXML/`, `data/thumbs/`, and `data/annotations/` with exactly the
95
+ inventory numbers the demo notebooks reference, so they run unchanged once extracted.
96
+
97
+ ## Installation
98
+
99
+ ```bash
100
+ poetry install
101
+ ```
102
+
103
+ Requires Python >=3.11,<3.15 -- `torch`'s `triton` dependency caps out at Python <3.15, so the
104
+ project's declared Python range matches that rather than the more typical `<4.0`.
105
+
106
+ ## Documentation
107
+
108
+ Built with Sphinx; requires the optional `docs` dependency group:
109
+
110
+ ```bash
111
+ poetry install --with docs
112
+ cd docs
113
+ make html
114
+ ```
@@ -0,0 +1,5 @@
1
+ """Tools for analysing PageXML/ATR transcriptions and scan images of archival documents:
2
+ detecting and splitting two-page book openings, clustering text lines and page layouts,
3
+ mining cross-page document-element sequences, ink-colour and missing-transcription detection,
4
+ and parsing EAD/METS archival finding-aid metadata.
5
+ """
@@ -0,0 +1,362 @@
1
+ """Allen Interval Algebra relations between PageXML regions (lines/regions treated as 1D
2
+ horizontal intervals).
3
+
4
+ `get_allen_interval` classifies a pair of regions into one of the 13 base Allen relations
5
+ (e.g. ``before``, ``meets``, ``overlaps``, ``equal``). `get_extended_allen_interval` further
6
+ splits the asymmetric ``overlaps``/``contains``/``starts``/``finishes`` relations by how much
7
+ of each region is covered (``m`` = most, ``l`` = least), giving the finer-grained
8
+ `EXTENDED_ALLEN_INTERVALS` vocabulary -- useful for distinguishing e.g. a small marginal
9
+ annotation overlapping a body-text line from two lines of comparable width overlapping evenly.
10
+ """
11
+
12
+ from collections import Counter
13
+
14
+ import pagexml.model.physical_document_model as pdm
15
+ from pagexml.helper.pagexml_helper import horizontal_group_lines
16
+
17
+
18
+ ALLEN_INTERVALS = {
19
+ 'after', 'meets', 'o',
20
+ 'fi', 'di', 's', 'equal', 'si', 'd', 'f',
21
+ 'oi', 'met by', 'before'
22
+ }
23
+
24
+ EXTENDED_ALLEN_INTERVALS = {
25
+ 'after', 'meets',
26
+ 'mom', 'mol', 'lom', 'lol',
27
+ 'm', 'mfi', 'lfi',
28
+ 'ldi', 'cdi', 'rdi',
29
+ 'ms', 'ls', 'equal', 'lsi', 'msi',
30
+ 'ld', 'cd', 'rd', 'lf', 'mf',
31
+ 'momi', 'moli', 'lomi', 'loli',
32
+ 'met by', 'before'
33
+ }
34
+
35
+
36
+ def get_doc_allen_interval_profile(doc: pdm.PageXMLRegion, use_extended: bool = True):
37
+ """Frequency count of Allen interval relations between consecutive horizontally-grouped
38
+ lines of `doc` (each line compared against the previous and next group), giving a profile
39
+ of how `doc`'s lines relate to their immediate neighbours. Uses
40
+ `get_extended_allen_interval` if `use_extended`, else the base `get_allen_interval`."""
41
+ interval_func = get_extended_allen_interval if use_extended else get_allen_interval
42
+ hor_group_lines = horizontal_group_lines(doc.get_lines())
43
+ prev_group = None
44
+ ai_freq = Counter()
45
+ for ci, curr_group in enumerate(hor_group_lines):
46
+ if prev_group is not None:
47
+ ai_freq.update([interval_func(pl, cl) for cl in curr_group for pl in prev_group])
48
+ if ci < len(hor_group_lines) - 1:
49
+ next_group = hor_group_lines[ci+1]
50
+ ai_freq.update([interval_func(cl, nl) for cl in curr_group for nl in next_group])
51
+ prev_group = curr_group
52
+ return ai_freq
53
+
54
+
55
+ def get_normalised_diff(x: int, y: int, max_diff: int = 0) -> int:
56
+ """`x - y`, snapped to 0 if the difference is within `max_diff` (treats near-identical
57
+ boundaries as exactly aligned)."""
58
+ diff = x - y
59
+ return 0 if abs(diff) <= max_diff else diff
60
+
61
+
62
+ def determine_overlap_relation(region1: pdm.PageXMLDoc, region2: pdm.PageXMLDoc) -> tuple[str | None, str | None]:
63
+ """For each region, whether the horizontal overlap with the other covers at least half
64
+ of its own width (``'m'``, most) or less (``'l'``, least). Returns `(None, None)` if the
65
+ regions don't overlap horizontally at all."""
66
+ overlap = pdm.get_horizontal_overlap(region1, region2)
67
+ if overlap == 0:
68
+ return None, None
69
+ relation1 = 'm' if overlap / region1.coords.width >= 0.5 else 'l'
70
+ relation2 = 'm' if overlap / region2.coords.width >= 0.5 else 'l'
71
+ return relation1, relation2
72
+
73
+
74
+ def get_normalised_region_diffs(region1: pdm.PageXMLDoc, region2: pdm.PageXMLDoc) -> tuple[int, int, int, int]:
75
+ """Left/right boundary differences between `region1` and `region2` (left-vs-left,
76
+ left-vs-right, right-vs-left, right-vs-right), using baselines if both regions have one,
77
+ else bounding boxes. These four values are what `get_allen_interval`/
78
+ `get_extended_allen_interval` branch on to determine the interval relation."""
79
+ if pdm.has_baseline(region1) and pdm.has_baseline(region2):
80
+ # only use baseline when both have them, otherwise, use bounding boxes
81
+ coords1 = region1.baseline
82
+ coords2 = region2.baseline
83
+ else:
84
+ coords1, coords2 = region1.coords, region2.coords
85
+ l1l2 = get_normalised_diff(coords1.left, coords2.left)
86
+ l1r2 = get_normalised_diff(coords1.left, coords2.right)
87
+ r1l2 = get_normalised_diff(coords1.right, coords2.left)
88
+ r1r2 = get_normalised_diff(coords1.right, coords2.right)
89
+ return l1l2, l1r2, r1l2, r1r2
90
+
91
+
92
+ def get_allen_interval(region1: pdm.PageXMLDoc, region2: pdm.PageXMLDoc,
93
+ debug: int = 0):
94
+ """Determine the (Extended) Allen Interval.
95
+
96
+ Use `max_diff` to work with thick boundaries.
97
+
98
+ :param region1: a PageXML document with spatial coordinates
99
+ :param region2: a PageXML document with spatial coordinates
100
+ :param debug: print debugging info
101
+ """
102
+ l1l2, l1r2, r1l2, r1r2 = get_normalised_region_diffs(region1, region2)
103
+ overlap1, overlap2 = determine_overlap_relation(region1, region2)
104
+ if debug > 0:
105
+ print(f"allen_interval.get_extended_allen_interval\n\tl1: left: {pdm.coords_as_span(region1.coords)}")
106
+ print(f"allen_interval.get_extended_allen_interval\n\tl2: left: {pdm.coords_as_span(region2.coords)}")
107
+ print(f"allen_interval.get_extended_allen_interval\n\tl1l2: {l1l2} r1r2: {r1r2} "
108
+ f"r1l2: {r1l2}\n\toverlap1: {overlap1} overlap2: {overlap2}")
109
+ if l1l2 < 0:
110
+ if r1r2 < 0:
111
+ if r1l2 < 0:
112
+ # --
113
+ # --
114
+ return 'after'
115
+ elif r1l2 == 0:
116
+ # ---
117
+ # ---
118
+ return 'meets'
119
+ else:
120
+ # ---
121
+ # -----
122
+ return 'o'
123
+ elif r1r2 == 0:
124
+ # ------
125
+ # ----
126
+ return "fi"
127
+ else:
128
+ # ------
129
+ # --
130
+ return 'di'
131
+ elif l1l2 == 0:
132
+ if r1r2 < 0:
133
+ # ----
134
+ # ------
135
+ return 's'
136
+ elif r1r2 == 0:
137
+ # no shift, complete overlap (equal)
138
+ # ------
139
+ # ------
140
+ return 'equal'
141
+ else:
142
+ # r1r2 > 0
143
+ # ------
144
+ # ----
145
+ return 'si'
146
+ else:
147
+ # l1l2 > 0
148
+ if r1r2 < 0:
149
+ # --
150
+ # ------
151
+ return 'd'
152
+ elif r1r2 == 0:
153
+ # ----
154
+ # ------
155
+ return 'f'
156
+ else:
157
+ # r1r2 > 0
158
+ if l1r2 < 0:
159
+ # -----
160
+ # ---
161
+ return 'oi'
162
+ elif l1r2 == 0:
163
+ # ---
164
+ # ---
165
+ return 'met by'
166
+ else:
167
+ # --
168
+ # --
169
+ return 'before'
170
+
171
+
172
+ def get_extended_allen_interval(region1: pdm.PageXMLDoc, region2: pdm.PageXMLDoc,
173
+ debug: int = 0):
174
+ """Determine the (Extended) Allen Interval.
175
+
176
+ Use `max_diff` to work with thick boundaries.
177
+
178
+ :param region1: a PageXML document with spatial coordinates
179
+ :param region2: a PageXML document with spatial coordinates
180
+ :param debug: print debugging info
181
+ """
182
+ l1l2, l1r2, r1l2, r1r2 = get_normalised_region_diffs(region1, region2)
183
+ overlap1, overlap2 = determine_overlap_relation(region1, region2)
184
+ if debug > 0:
185
+ print(f"allen_interval.get_extended_allen_interval\n\tl1: left: {pdm.coords_as_span(region1.coords)}")
186
+ print(f"allen_interval.get_extended_allen_interval\n\tl2: left: {pdm.coords_as_span(region2.coords)}")
187
+ print(f"allen_interval.get_extended_allen_interval\n\tl1l2: {l1l2} r1r2: {r1r2} "
188
+ f"r1l2: {r1l2}\n\toverlap1: {overlap1} overlap2: {overlap2}")
189
+ if l1l2 < 0:
190
+ if r1r2 < 0:
191
+ if r1l2 < 0:
192
+ # --
193
+ # --
194
+ return 'after'
195
+ elif r1l2 == 0:
196
+ # ---
197
+ # ---
198
+ return 'meets'
199
+ else:
200
+ if overlap1 == 'l' and overlap2 == 'm':
201
+ # right-shift, shrink, most part 2 overlapped by least part 1 (lom)
202
+ # -----
203
+ # ---
204
+ return 'lom'
205
+ elif overlap1 == 'm' and overlap2 == 'l':
206
+ # right-shift, stretch, most part 1 overlaps least part 2 (mol)
207
+ # ---
208
+ # -----
209
+ return 'mol'
210
+ elif overlap1 == 'm' and overlap2 == 'm':
211
+ # right-shift, most part overlaps most part (mom)
212
+ # -----
213
+ # -----
214
+ return 'mom'
215
+ elif overlap1 == 'l' and overlap2 == 'l':
216
+ # right-shift, least part overlaps least part (lol)
217
+ # ----
218
+ # ---
219
+ return 'lol'
220
+ else:
221
+ raise ValueError(f"unexpected situation - l1l2: {l1l2} r1r2: {r1r2} "
222
+ f"r1l2: {r1l2} overlap1: {overlap1} overlap2: {overlap2}")
223
+ elif r1r2 == 0:
224
+ if overlap1 == 'm':
225
+ # right-shift, finished by, most part overlap (mfi)
226
+ # ------
227
+ # ----
228
+ return 'mfi'
229
+ elif overlap1 == 'l':
230
+ # right-shift, finished by, least part overlap (lfi)
231
+ # ------
232
+ # --
233
+ return "lfi"
234
+ else:
235
+ raise ValueError(f"unexpected situation - l1l2: {l1l2} r1r2: {r1r2} "
236
+ f"r1l2: {r1l2} l1r2: {l1r2} "
237
+ f"overlap1: {overlap1} overlap2: {overlap2}")
238
+ else:
239
+ diff = get_normalised_diff(abs(l1l2), abs(r1r2))
240
+ if diff < 0:
241
+ # right-shift, contained by, right-aligned
242
+ # ------
243
+ # --
244
+ return 'ldi'
245
+ elif diff == 0:
246
+ # no shift, contained by, center-aligned
247
+ # ------
248
+ # ----
249
+ return 'cdi'
250
+ else:
251
+ # left-shift, contained by, left-aligned
252
+ # ------
253
+ # --
254
+ return 'rdi'
255
+ elif l1l2 == 0:
256
+ if r1r2 < 0:
257
+ if overlap2 == 'm':
258
+ # right-shift, starts, overlaps with most part (ms)
259
+ # ----
260
+ # ------
261
+ return 'ms'
262
+ elif overlap2 == 'l':
263
+ # right-shift, starts, overlaps with least part (ls)
264
+ # --
265
+ # ------
266
+ return 'ls'
267
+ else:
268
+ raise ValueError(f"unexpected situation - l1l2: {l1l2} r1r2: {r1r2} "
269
+ f"r1l2: {r1l2} l1r2: {l1r2} "
270
+ f"overlap1: {overlap1} overlap2: {overlap2}")
271
+ elif r1r2 == 0:
272
+ # no shift, complete overlap (equal)
273
+ # ------
274
+ # ------
275
+ return 'equal'
276
+ else:
277
+ # r1r2 > 0
278
+ if overlap1 == 'm':
279
+ # left-shift, starts, most part overlaps
280
+ # ------
281
+ # ----
282
+ return 'msi'
283
+ elif overlap1 == 'l':
284
+ # left-shift, starts, least part overlaps
285
+ # ------
286
+ # --
287
+ return 'lsi'
288
+ else:
289
+ raise ValueError(f"unexpected situation - l1l2: {l1l2} r1r2: {r1r2} "
290
+ f"r1l2: {r1l2} l1r2: {l1r2} "
291
+ f"overlap1: {overlap1} overlap2: {overlap2}")
292
+ else:
293
+ # l1l2 > 0
294
+ if r1r2 < 0:
295
+ diff = get_normalised_diff(abs(l1l2), abs(r1r2))
296
+ if diff < 0:
297
+ # right-shift, contains, left-aligned
298
+ # --
299
+ # ------
300
+ return 'ld'
301
+ elif diff == 0:
302
+ # no shift, contains, center-aligned
303
+ # ----
304
+ # ------
305
+ return 'cd'
306
+ else:
307
+ # left-shift, contains, right-aligned
308
+ # --
309
+ # ------
310
+ return 'rd'
311
+ elif r1r2 == 0:
312
+ if overlap2 == 'm':
313
+ # left-shift, finishes, most part overlap (mfi)
314
+ # ----
315
+ # ------
316
+ return 'mf'
317
+ elif overlap2 == 'l':
318
+ # left-shift, finishes, least part overlap (lfi)
319
+ # --
320
+ # ------
321
+ return 'lf'
322
+ else:
323
+ raise ValueError(f"unexpected situation - l1l2: {l1l2} r1r2: {r1r2} "
324
+ f"r1l2: {r1l2} l1r2: {l1r2} "
325
+ f"overlap1: {overlap1} overlap2: {overlap2}")
326
+ else:
327
+ # r1r2 > 0
328
+ if l1r2 < 0:
329
+ if overlap1 == 'l' and overlap2 == 'm':
330
+ # left-shift, shrink, least part 2 overlapped by most part 1 (lomi)
331
+ # -----
332
+ # ---
333
+ return 'lomi'
334
+ elif overlap1 == 'm' and overlap2 == 'l':
335
+ # left-shift, stretch, least part 2 overlaps most part 1 (moli)
336
+ # ---
337
+ # -----
338
+ return 'moli'
339
+ elif overlap1 == 'm' and overlap2 == 'm':
340
+ # left-shift, most part overlaps most part (momi)
341
+ # -----
342
+ # -----
343
+ return 'momi'
344
+ elif overlap1 == 'l' and overlap2 == 'l':
345
+ # left-shift, least part overlaps least part (loli)
346
+ # ---
347
+ # ----
348
+ return 'loli'
349
+ else:
350
+ print(f"region 1: {region1.id} {region1.coords.box_string}")
351
+ print(f"region 2: {region2.id} {region2.coords.box_string}")
352
+ raise ValueError(f"unexpected situation - l1l2: {l1l2} r1r2: {r1r2} "
353
+ f"r1l2: {r1l2} l1r2: {l1r2} "
354
+ f"overlap1: {overlap1} overlap2: {overlap2}")
355
+ elif l1r2 == 0:
356
+ # ---
357
+ # ---
358
+ return 'met by'
359
+ else:
360
+ # --
361
+ # --
362
+ return 'before'