archival-structures 0.1.0__py3-none-any.whl

archival_structures/__init__.py

@@ -0,0 +1,5 @@
+"""Tools for analysing PageXML/ATR transcriptions and scan images of archival documents:
+detecting and splitting two-page book openings, clustering text lines and page layouts,
+mining cross-page document-element sequences, ink-colour and missing-transcription detection,
+and parsing EAD/METS archival finding-aid metadata.
+"""

archival_structures/analysis/allen_interval.py

@@ -0,0 +1,362 @@
+"""Allen Interval Algebra relations between PageXML regions (lines/regions treated as 1D
+horizontal intervals).
+
+`get_allen_interval` classifies a pair of regions into one of the 13 base Allen relations
+(e.g. ``before``, ``meets``, ``overlaps``, ``equal``). `get_extended_allen_interval` further
+splits the asymmetric ``overlaps``/``contains``/``starts``/``finishes`` relations by how much
+of each region is covered (``m`` = most, ``l`` = least), giving the finer-grained
+`EXTENDED_ALLEN_INTERVALS` vocabulary -- useful for distinguishing e.g. a small marginal
+annotation overlapping a body-text line from two lines of comparable width overlapping evenly.
+"""
+
+from collections import Counter
+
+import pagexml.model.physical_document_model as pdm
+from pagexml.helper.pagexml_helper import horizontal_group_lines
+
+
+ALLEN_INTERVALS = {
+    'after', 'meets', 'o',
+    'fi', 'di', 's', 'equal', 'si', 'd', 'f',
+    'oi', 'met by', 'before'
+}
+
+EXTENDED_ALLEN_INTERVALS = {
+    'after', 'meets',
+    'mom', 'mol', 'lom', 'lol',
+    'm', 'mfi', 'lfi',
+    'ldi', 'cdi', 'rdi',
+    'ms', 'ls', 'equal', 'lsi', 'msi',
+    'ld', 'cd', 'rd', 'lf', 'mf',
+    'momi', 'moli', 'lomi', 'loli',
+    'met by', 'before'
+}
+
+
+def get_doc_allen_interval_profile(doc: pdm.PageXMLRegion, use_extended: bool = True):
+    """Frequency count of Allen interval relations between consecutive horizontally-grouped
+    lines of `doc` (each line compared against the previous and next group), giving a profile
+    of how `doc`'s lines relate to their immediate neighbours. Uses
+    `get_extended_allen_interval` if `use_extended`, else the base `get_allen_interval`."""
+    interval_func = get_extended_allen_interval if use_extended else get_allen_interval
+    hor_group_lines = horizontal_group_lines(doc.get_lines())
+    prev_group = None
+    ai_freq = Counter()
+    for ci, curr_group in enumerate(hor_group_lines):
+        if prev_group is not None:
+            ai_freq.update([interval_func(pl, cl) for cl in curr_group for pl in prev_group])
+        if ci < len(hor_group_lines) - 1:
+            next_group = hor_group_lines[ci+1]
+            ai_freq.update([interval_func(cl, nl) for cl in curr_group for nl in next_group])
+        prev_group = curr_group
+    return ai_freq
+
+
+def get_normalised_diff(x: int, y: int, max_diff: int = 0) -> int:
+    """`x - y`, snapped to 0 if the difference is within `max_diff` (treats near-identical
+    boundaries as exactly aligned)."""
+    diff = x - y
+    return 0 if abs(diff) <= max_diff else diff
+
+
+def determine_overlap_relation(region1: pdm.PageXMLDoc, region2: pdm.PageXMLDoc) -> tuple[str | None, str | None]:
+    """For each region, whether the horizontal overlap with the other covers at least half
+    of its own width (``'m'``, most) or less (``'l'``, least). Returns `(None, None)` if the
+    regions don't overlap horizontally at all."""
+    overlap = pdm.get_horizontal_overlap(region1, region2)
+    if overlap == 0:
+        return None, None
+    relation1 = 'm' if overlap / region1.coords.width >= 0.5 else 'l'
+    relation2 = 'm' if overlap / region2.coords.width >= 0.5 else 'l'
+    return relation1, relation2
+
+
+def get_normalised_region_diffs(region1: pdm.PageXMLDoc, region2: pdm.PageXMLDoc) -> tuple[int, int, int, int]:
+    """Left/right boundary differences between `region1` and `region2` (left-vs-left,
+    left-vs-right, right-vs-left, right-vs-right), using baselines if both regions have one,
+    else bounding boxes. These four values are what `get_allen_interval`/
+    `get_extended_allen_interval` branch on to determine the interval relation."""
+    if pdm.has_baseline(region1) and pdm.has_baseline(region2):
+        # only use baseline when both have them, otherwise, use bounding boxes
+        coords1 = region1.baseline
+        coords2 = region2.baseline
+    else:
+        coords1, coords2 = region1.coords, region2.coords
+    l1l2 = get_normalised_diff(coords1.left, coords2.left)
+    l1r2 = get_normalised_diff(coords1.left, coords2.right)
+    r1l2 = get_normalised_diff(coords1.right, coords2.left)
+    r1r2 = get_normalised_diff(coords1.right, coords2.right)
+    return l1l2, l1r2, r1l2, r1r2
+
+
+def get_allen_interval(region1: pdm.PageXMLDoc, region2: pdm.PageXMLDoc,
+                       debug: int = 0):
+    """Determine the (Extended) Allen Interval.
+
+    Use `max_diff` to work with thick boundaries.
+
+    :param region1: a PageXML document with spatial coordinates
+    :param region2: a PageXML document with spatial coordinates
+    :param debug: print debugging info
+    """
+    l1l2, l1r2, r1l2, r1r2 = get_normalised_region_diffs(region1, region2)
+    overlap1, overlap2 = determine_overlap_relation(region1, region2)
+    if debug > 0:
+        print(f"allen_interval.get_extended_allen_interval\n\tl1: left: {pdm.coords_as_span(region1.coords)}")
+        print(f"allen_interval.get_extended_allen_interval\n\tl2: left: {pdm.coords_as_span(region2.coords)}")
+        print(f"allen_interval.get_extended_allen_interval\n\tl1l2: {l1l2}  r1r2: {r1r2}  "
+              f"r1l2: {r1l2}\n\toverlap1: {overlap1}  overlap2: {overlap2}")
+    if l1l2 < 0:
+        if r1r2 < 0:
+            if r1l2 < 0:
+                # --
+                #     --
+                return 'after'
+            elif r1l2 == 0:
+                # ---
+                #    ---
+                return 'meets'
+            else:
+                # ---
+                #  -----
+                return 'o'
+        elif r1r2 == 0:
+            # ------
+            #   ----
+            return "fi"
+        else:
+            # ------
+            #  --
+            return 'di'
+    elif l1l2 == 0:
+        if r1r2 < 0:
+            # ----
+            # ------
+            return 's'
+        elif r1r2 == 0:
+            # no shift, complete overlap (equal)
+            # ------
+            # ------
+            return 'equal'
+        else:
+            # r1r2 > 0
+            # ------
+            # ----
+            return 'si'
+    else:
+        # l1l2 > 0
+        if r1r2 < 0:
+            #  --
+            # ------
+            return 'd'
+        elif r1r2 == 0:
+            #   ----
+            # ------
+            return 'f'
+        else:
+            # r1r2 > 0
+            if l1r2 < 0:
+                #  -----
+                # ---
+                return 'oi'
+            elif l1r2 == 0:
+                #    ---
+                # ---
+                return 'met by'
+            else:
+                #     --
+                # --
+                return 'before'
+
+
+def get_extended_allen_interval(region1: pdm.PageXMLDoc, region2: pdm.PageXMLDoc,
+                                debug: int = 0):
+    """Determine the (Extended) Allen Interval.
+
+    Use `max_diff` to work with thick boundaries.
+
+    :param region1: a PageXML document with spatial coordinates
+    :param region2: a PageXML document with spatial coordinates
+    :param debug: print debugging info
+    """
+    l1l2, l1r2, r1l2, r1r2 = get_normalised_region_diffs(region1, region2)
+    overlap1, overlap2 = determine_overlap_relation(region1, region2)
+    if debug > 0:
+        print(f"allen_interval.get_extended_allen_interval\n\tl1: left: {pdm.coords_as_span(region1.coords)}")
+        print(f"allen_interval.get_extended_allen_interval\n\tl2: left: {pdm.coords_as_span(region2.coords)}")
+        print(f"allen_interval.get_extended_allen_interval\n\tl1l2: {l1l2}  r1r2: {r1r2}  "
+              f"r1l2: {r1l2}\n\toverlap1: {overlap1}  overlap2: {overlap2}")
+    if l1l2 < 0:
+        if r1r2 < 0:
+            if r1l2 < 0:
+                # --
+                #    --
+                return 'after'
+            elif r1l2 == 0:
+                # ---
+                #    ---
+                return 'meets'
+            else:
+                if overlap1 == 'l' and overlap2 == 'm':
+                    # right-shift, shrink, most part 2 overlapped by least part 1 (lom)
+                    # -----
+                    #    ---
+                    return 'lom'
+                elif overlap1 == 'm' and overlap2 == 'l':
+                    # right-shift, stretch, most part 1 overlaps least part 2 (mol)
+                    # ---
+                    #  -----
+                    return 'mol'
+                elif overlap1 == 'm' and overlap2 == 'm':
+                    # right-shift, most part overlaps most part (mom)
+                    # -----
+                    #  -----
+                    return 'mom'
+                elif overlap1 == 'l' and overlap2 == 'l':
+                    # right-shift, least part overlaps least part (lol)
+                    # ----
+                    #    ---
+                    return 'lol'
+                else:
+                    raise ValueError(f"unexpected situation - l1l2: {l1l2}  r1r2: {r1r2}  "
+                                     f"r1l2: {r1l2}  overlap1: {overlap1}  overlap2: {overlap2}")
+        elif r1r2 == 0:
+            if overlap1 == 'm':
+                # right-shift, finished by, most part overlap (mfi)
+                # ------
+                #   ----
+                return 'mfi'
+            elif overlap1 == 'l':
+                # right-shift, finished by, least part overlap (lfi)
+                # ------
+                #     --
+                return "lfi"
+            else:
+                raise ValueError(f"unexpected situation - l1l2: {l1l2}  r1r2: {r1r2}  "
+                                 f"r1l2: {r1l2}  l1r2: {l1r2}  "
+                                 f"overlap1: {overlap1}  overlap2: {overlap2}")
+        else:
+            diff = get_normalised_diff(abs(l1l2), abs(r1r2))
+            if diff < 0:
+                # right-shift, contained by, right-aligned
+                # ------
+                #  --
+                return 'ldi'
+            elif diff == 0:
+                # no shift, contained by, center-aligned
+                # ------
+                #  ----
+                return 'cdi'
+            else:
+                # left-shift, contained by, left-aligned
+                # ------
+                #    --
+                return 'rdi'
+    elif l1l2 == 0:
+        if r1r2 < 0:
+            if overlap2 == 'm':
+                # right-shift, starts, overlaps with most part (ms)
+                # ----
+                # ------
+                return 'ms'
+            elif overlap2 == 'l':
+                # right-shift, starts, overlaps with least part (ls)
+                # --
+                # ------
+                return 'ls'
+            else:
+                raise ValueError(f"unexpected situation - l1l2: {l1l2}  r1r2: {r1r2}  "
+                                 f"r1l2: {r1l2}  l1r2: {l1r2}  "
+                                 f"overlap1: {overlap1}  overlap2: {overlap2}")
+        elif r1r2 == 0:
+            # no shift, complete overlap (equal)
+            # ------
+            # ------
+            return 'equal'
+        else:
+            # r1r2 > 0
+            if overlap1 == 'm':
+                # left-shift, starts, most part overlaps
+                # ------
+                # ----
+                return 'msi'
+            elif overlap1 == 'l':
+                # left-shift, starts, least part overlaps
+                # ------
+                # --
+                return 'lsi'
+            else:
+                raise ValueError(f"unexpected situation - l1l2: {l1l2}  r1r2: {r1r2}  "
+                                 f"r1l2: {r1l2}  l1r2: {l1r2}  "
+                                 f"overlap1: {overlap1}  overlap2: {overlap2}")
+    else:
+        # l1l2 > 0
+        if r1r2 < 0:
+            diff = get_normalised_diff(abs(l1l2), abs(r1r2))
+            if diff < 0:
+                # right-shift, contains, left-aligned
+                #  --
+                # ------
+                return 'ld'
+            elif diff == 0:
+                # no shift, contains, center-aligned
+                #  ----
+                # ------
+                return 'cd'
+            else:
+                # left-shift, contains, right-aligned
+                #    --
+                # ------
+                return 'rd'
+        elif r1r2 == 0:
+            if overlap2 == 'm':
+                # left-shift, finishes, most part overlap (mfi)
+                #   ----
+                # ------
+                return 'mf'
+            elif overlap2 == 'l':
+                # left-shift, finishes, least part overlap (lfi)
+                #     --
+                # ------
+                return 'lf'
+            else:
+                raise ValueError(f"unexpected situation - l1l2: {l1l2}  r1r2: {r1r2}  "
+                                 f"r1l2: {r1l2}  l1r2: {l1r2}  "
+                                 f"overlap1: {overlap1}  overlap2: {overlap2}")
+        else:
+            # r1r2 > 0
+            if l1r2 < 0:
+                if overlap1 == 'l' and overlap2 == 'm':
+                    # left-shift, shrink, least part 2 overlapped by most part 1 (lomi)
+                    #  -----
+                    # ---
+                    return 'lomi'
+                elif overlap1 == 'm' and overlap2 == 'l':
+                    # left-shift, stretch, least part 2 overlaps most part 1 (moli)
+                    #    ---
+                    # -----
+                    return 'moli'
+                elif overlap1 == 'm' and overlap2 == 'm':
+                    # left-shift, most part overlaps most part (momi)
+                    #  -----
+                    # -----
+                    return 'momi'
+                elif overlap1 == 'l' and overlap2 == 'l':
+                    # left-shift, least part overlaps least part (loli)
+                    #    ---
+                    # ----
+                    return 'loli'
+                else:
+                    print(f"region 1: {region1.id} {region1.coords.box_string}")
+                    print(f"region 2: {region2.id} {region2.coords.box_string}")
+                    raise ValueError(f"unexpected situation - l1l2: {l1l2}  r1r2: {r1r2}  "
+                                     f"r1l2: {r1l2}  l1r2: {l1r2}  "
+                                     f"overlap1: {overlap1}  overlap2: {overlap2}")
+            elif l1r2 == 0:
+                #    ---
+                # ---
+                return 'met by'
+            else:
+                #     --
+                # --
+                return 'before'

archival_structures/analysis/grid_analysis.py

@@ -0,0 +1,254 @@
+"""Grid-based page-layout fingerprinting.
+
+Each page in a `doc_set` is rasterised onto a shared `unit_size`-pixel grid (one cell is "on"
+wherever a text line falls), then `GridPattern` slides a small `pattern_size` x `pattern_size`
+window across that grid and counts how often each local on/off pattern occurs, both
+per-document and corpus-wide. The resulting per-document TF-IDF vector (`GridPattern.tfidf`)
+is a layout fingerprint that `archival_structures.analysis.page_layout_clustering` clusters
+with HDBSCAN -- two pages with a similar arrangement of text lines get similar fingerprints,
+regardless of the actual text content.
+"""
+
+import pickle
+import time
+from collections import Counter, defaultdict
+from typing import List, Set, Union
+
+import numpy as np
+import pagexml.model.physical_document_model as pdm
+import pagexml.analysis.layout_stats as layout
+from numpy.lib.stride_tricks import sliding_window_view
+
+
+def get_doc_set_grid_points(doc_set: Union[List[pdm.PageXMLRegion], Set[pdm.PageXMLRegion]],
+                            unit_size: int) -> dict[tuple[int, int], int]:
+    """All `(x, y)` grid coordinates (spaced `unit_size` pixels apart) covering the largest
+    page in `doc_set`, mapped to a dense integer index -- the shared coordinate system every
+    page in the set is rasterised onto."""
+    max_x, max_y = get_interpolated_max_x_y(doc_set, unit_size=unit_size)
+    grid_points = [(x, y) for x in range(0, max_x, unit_size) for y in range(0, max_y, unit_size)]
+    print("done setting base grid points")
+
+    return {point: pi for pi, point in enumerate(grid_points)}
+
+
+def get_interpolated_max_x_y(doc_set: Union[List[pdm.PageXMLRegion], Set[pdm.PageXMLRegion]],
+                             unit_size: int = 50) -> tuple[int, int]:
+    """The largest page width/height in `doc_set`, rounded to the nearest `unit_size`."""
+    max_x = max(page.coords.right for page in doc_set)
+    max_y = max(page.coords.bottom for page in doc_set)
+
+    max_x = int(np.round(max_x / unit_size, 0) * unit_size)
+    max_y = int(np.round(max_y / unit_size, 0) * unit_size)
+    return max_x, max_y
+
+
+def doc_grid_points_to_vectors(doc_grid_points: np.array) -> np.array:
+    """Flatten a `(num_docs, range_x, range_y)` grid-point array into `(num_docs, range_x *
+    range_y)`, one row per document."""
+    dims = doc_grid_points.shape
+    if len(dims) != 3:
+        raise IndexError(f"invalid shape for doc_grid_pionts: {dims}, must be a 3D array.")
+    return np.resize(doc_grid_points, (dims[0], dims[1] * dims[2]))
+
+
+def time_step(start, prev_step):
+    """Print elapsed time since `prev_step` and since `start`, returning the current time
+    (for chaining: `step = time_step(start, step)`)."""
+    curr_step = time.time()
+    took = curr_step - prev_step
+    total = curr_step - start
+    print(f"took {took: >.1f} seconds (total: {total: >.1f})")
+    return curr_step
+
+
+def get_line_grid_left_right(line: pdm.PageXMLTextLine, unit_size: int = 50, indent: int = 0):
+    """`line`'s left/right extent (from its baseline if available, else its coordinates),
+    snapped to the nearest `unit_size` grid line. `indent` shifts the snapped boundary inward
+    on both sides before rounding."""
+    if line.baseline is not None and line.baseline.points is not None:
+        left = line.baseline.left
+        right = line.baseline.right
+    elif line.coords is not None and line.coords.points is not None:
+        left = line.coords.left
+        right = line.coords.right
+    else:
+        return None, None
+    # print(f"original left: {left}\toriginal right: {right}")
+    left += indent
+    dist_from_prev = left % unit_size
+    dist_from_next = unit_size - dist_from_prev
+    # print(f"left dist_from_next: {dist_from_next}")
+    left = left - dist_from_prev if dist_from_next > unit_size / 4 else left + dist_from_next
+    right -= indent
+    dist_from_prev = right % unit_size
+    dist_from_next = unit_size - dist_from_prev
+    # print(f"right dist_from_next: {dist_from_next}")
+    right = right - dist_from_prev if dist_from_prev < unit_size / 4 else right + dist_from_next
+    return left, right
+
+
+def get_line_grid_top_bottom(line: pdm.PageXMLTextLine, unit_size: int = 50):
+    """`line`'s top/bottom extent (from its baseline and x-height if available, else a
+    margin-trimmed fraction of its bounding box), snapped to the nearest `unit_size` grid
+    line."""
+    if line.baseline is not None and line.baseline.points is not None:
+        if line.xheight is not None:
+            top = line.baseline.top - line.xheight
+        else:
+            top = line.coords.top + int(line.coords.height * 0.25)
+        bottom = line.baseline.bottom
+    elif line.coords is not None and line.coords.points is not None:
+        top = line.coords.top + int(line.coords.height * 0.25)
+        bottom = line.coords.bottom - int(line.coords.height * 0.25)
+    else:
+        return None
+    # print(f"original bottom: {bottom}\toriginal top: {top}")
+    dist_from_prev = top % unit_size
+    dist_from_next = unit_size - dist_from_prev
+    top = top - dist_from_prev if dist_from_next > unit_size / 4 else top + dist_from_next
+    dist_from_prev = bottom % unit_size
+    dist_from_next = unit_size - dist_from_prev
+    bottom = bottom - dist_from_prev if dist_from_prev < unit_size / 4 else bottom + dist_from_next
+    return top, bottom
+
+
+def get_line_grid_points(doc_set: Union[List[pdm.PageXMLRegion], Set[pdm.PageXMLRegion]],
+                         unit_size: int = 50, indent: int = 0) -> np.array:
+    """Rasterise every document in `doc_set` onto a shared `unit_size`-pixel grid: a
+    `(num_docs, range_x, range_y)` array where a cell is 1.0 wherever any text line's
+    (snapped) bounding box falls, 0.0 elsewhere."""
+    max_x, max_y = get_interpolated_max_x_y(doc_set, unit_size=unit_size)
+    range_x, range_y = int(max_x / unit_size), int(max_y / unit_size)
+
+    doc_grid_points = np.zeros((len(doc_set), range_x, range_y))
+
+    for di, doc in enumerate(doc_set):
+        for line in doc.get_lines():
+            left, right = get_line_grid_left_right(line, unit_size=unit_size, indent=indent)
+            top, bottom = get_line_grid_top_bottom(line, unit_size=unit_size)
+            x_start, x_end = int(left / unit_size), int(right / unit_size)
+            y_start, y_end = int(top / unit_size), int(bottom / unit_size)
+            doc_grid_points[di][x_start:x_end, y_start:y_end] = 1.0
+    print(f"done setting line grid_points for {len(doc_set)} docs, "
+          f"with {range_x} x and {range_y} y points")
+    return doc_grid_points
+
+
+def get_region_grid_points(doc_set: Union[List[pdm.PageXMLRegion], Set[pdm.PageXMLRegion]],
+                           unit_size: int = 50) -> np.array:
+    """Like `get_line_grid_points`, but rasterised from text *region* bounding boxes rather
+    than individual lines -- coarser, since region detection tends to merge visually distinct
+    elements (prefer `get_line_grid_points` for layout fingerprinting)."""
+    positions = ['left', 'top', 'right', 'bottom']
+    max_x, max_y = get_interpolated_max_x_y(doc_set, unit_size=unit_size)
+    range_x, range_y = int(max_x / unit_size), int(max_y / unit_size)
+
+    doc_grid_points = np.zeros((len(doc_set), range_x, range_y))
+
+    for di, doc in enumerate(doc_set):
+        for tr in doc.get_textual_regions():
+            tr_pos = {}
+            for pos in positions:
+                tr_pos[pos] = int(np.round(tr.coords.__getattribute__(pos) / unit_size, 0))
+            x_start, x_end = tr_pos['left'], tr_pos['right']
+            y_start, y_end = tr_pos['top'], tr_pos['bottom']
+            doc_grid_points[di][x_start:x_end, y_start:y_end] = 1.0
+    print(f"done setting region grid_points for {len(doc_set)} docs, "
+          f"with {range_x} x and {range_y} y points")
+    return doc_grid_points
+
+
+class GridPattern:
+    """Per-document layout fingerprints, built by rasterising each document's text lines onto
+    a shared grid (`get_line_grid_points`) and counting local `pattern_size` x `pattern_size`
+    on/off patterns across that grid, both per-document and corpus-wide. `self.tfidf` (shape
+    `(num_docs, vocab_size)`) is the resulting fingerprint matrix, suitable for clustering or
+    nearest-neighbour search; `self.doc_pfidf(doc_id)` looks up one document's row by id."""
+
+    def __init__(self, doc_set: Union[List[pdm.PageXMLRegion], Set[pdm.PageXMLRegion]],
+                 unit_size: int = 50, pattern_size: int = 3):
+        # self.doc_set = doc_set
+        self.doc_ids = [doc.id for doc in doc_set]
+        self.id2idx = {doc.id: di for di, doc in enumerate(doc_set)}
+        self.idx2id = {di: doc.id for di, doc in enumerate(doc_set)}
+        self.num_docs = len(doc_set)
+        self.unit_size = unit_size
+        self.pattern_size = pattern_size
+        start = time.time()
+        self.grid_points = get_doc_set_grid_points(doc_set, unit_size=unit_size)
+        step = time_step(start, start)
+        self.doc_grid_points = get_line_grid_points(doc_set, unit_size)
+        step = time_step(start, step)
+        self.coll_pattern_freq = Counter()
+        self.doc_pattern_freq = defaultdict(Counter)
+        self.pattern_doc_freq = Counter()
+        self._compute_pattern_freq()
+        step = time_step(start, step)
+        self.vocab = {pattern: pi for pi, pattern in enumerate(self.pattern_doc_freq.keys())}
+        self.vocab_inv = {v: k for k, v in self.vocab.items()}
+        self.vocab_size = len(self.vocab)
+        self.tf = None
+        self.tfidf = self._compute_pattern_tfidf()
+        time_step(start, step)
+
+    def doc_pf(self, doc_id):
+        """Raw local-pattern frequency counter (`Counter`) for the document with id `doc_id`."""
+        if doc_id not in self.id2idx:
+            raise KeyError(f"unknown doc_id {doc_id}")
+        return self.doc_pattern_freq[self.id2idx[doc_id]]
+
+    def doc_pfidf(self, doc_id):
+        """TF-IDF layout-fingerprint vector (shape `(vocab_size,)`) for the document with id
+        `doc_id` -- a row of `self.tfidf`."""
+        if doc_id not in self.id2idx:
+            raise KeyError(f"unknown doc_id {doc_id}")
+        return self.tfidf[self.id2idx[doc_id]]
+
+    def _compute_pattern_freq(self):
+        window_shape = (self.pattern_size, self.pattern_size)
+        for di in range(0, self.num_docs):
+            windows = sliding_window_view(self.doc_grid_points[di], window_shape)
+            patterns = np.reshape(windows, (windows.shape[0] * windows.shape[1], self.pattern_size**2))
+            patterns = [tuple(pattern) for pattern in patterns]
+            # patterns = np.array([(2 ** (np.where(pattern)[0]+1)).sum() for pattern in patterns])
+            self.doc_pattern_freq[di].update(patterns)
+            self.coll_pattern_freq.update(patterns)
+
+        for doc_id in self.doc_pattern_freq:
+            self.pattern_doc_freq.update([pattern for pattern in self.doc_pattern_freq[doc_id]])
+        print(f"done computing patterns, {len(self.coll_pattern_freq)} distinct patterns")
+
+    def _compute_pattern_tfidf(self):
+        self.tf = np.zeros((self.num_docs, self.vocab_size))
+        tf_idf = np.zeros((self.num_docs, self.vocab_size))
+        for di in self.doc_pattern_freq:
+            doc_length = sum(self.doc_pattern_freq[di].values())
+            tf_idf[di] = np.zeros(self.vocab_size)
+            for pattern, tf in self.doc_pattern_freq[di].most_common():
+                prob_tf = tf / doc_length
+                log_tf = 1 + np.log(tf)
+                cf = self.coll_pattern_freq[pattern]
+                df = self.pattern_doc_freq[pattern]
+                # idf = np.log((self.num_docs - df) / df)
+                idf = np.log(self.num_docs / df)
+                if idf < 0.0:
+                    print(tf, cf, df, self.num_docs, self.num_docs - df + 1, idf)
+                pattern_idx = self.vocab[pattern]
+                self.tf[di][pattern_idx] = prob_tf
+                tf_idf[di][pattern_idx] = log_tf * idf
+
+        print("done computing tf-idf of patterns")
+        return tf_idf
+
+    def save(self, filename: str):
+        """Pickle this `GridPattern` to `filename`."""
+        with open(filename, 'wb') as fh:
+            pickle.dump(self, fh)
+
+    @staticmethod
+    def load(filename: str) -> 'GridPattern':
+        """Load a `GridPattern` previously written with `save`."""
+        with open(filename, 'rb') as fh:
+            grid_pattern = pickle.load(fh)
+        return grid_pattern