ocr-stringdist 0.2.2__cp313-cp313-musllinux_1_1_x86_64.whl → 1.0.0__cp313-cp313-musllinux_1_1_x86_64.whl

ocr_stringdist/__init__.py

@@ -1,17 +1,11 @@
 from .default_ocr_distances import ocr_distance_map
-from .levenshtein import (
-    WeightedLevenshtein,
-    batch_weighted_levenshtein_distance,
-    explain_weighted_levenshtein,
-    weighted_levenshtein_distance,
-)
+from .learner import CostLearner
+from .levenshtein import WeightedLevenshtein
 from .matching import find_best_candidate
 
 __all__ = [
     "ocr_distance_map",
+    "CostLearner",
     "WeightedLevenshtein",
-    "weighted_levenshtein_distance",
-    "batch_weighted_levenshtein_distance",
-    "explain_weighted_levenshtein",
     "find_best_candidate",
 ]

ocr_stringdist/edit_operation.py

@@ -0,0 +1,16 @@
+from dataclasses import dataclass
+from typing import Literal, Optional
+
+OperationType = Literal["substitute", "insert", "delete", "match"]
+
+
+@dataclass(frozen=True)
+class EditOperation:
+    """
+    Represents a single edit operation (substitution, insertion, deletion or match).
+    """
+
+    op_type: OperationType
+    source_token: Optional[str]
+    target_token: Optional[str]
+    cost: float

ocr_stringdist/learner.py

@@ -0,0 +1,254 @@
+import itertools
+import math
+from collections import defaultdict
+from collections.abc import Iterable
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Callable, Optional
+
+if TYPE_CHECKING:
+    from .edit_operation import EditOperation
+    from .levenshtein import WeightedLevenshtein
+    from .protocols import Aligner
+
+CostFunction = Callable[[float], float]
+
+
+def negative_log_likelihood(probability: float) -> float:
+    if probability <= 0.0:
+        raise ValueError("Probability must be positive to compute negative log likelihood.")
+    return -math.log(probability)
+
+
+@dataclass
+class TallyCounts:
+    substitutions: defaultdict[tuple[str, str], int] = field(
+        default_factory=lambda: defaultdict(int)
+    )
+    insertions: defaultdict[str, int] = field(default_factory=lambda: defaultdict(int))
+    deletions: defaultdict[str, int] = field(default_factory=lambda: defaultdict(int))
+    source_chars: defaultdict[str, int] = field(default_factory=lambda: defaultdict(int))
+    target_chars: defaultdict[str, int] = field(default_factory=lambda: defaultdict(int))
+    vocab: set[str] = field(default_factory=set)
+
+
+@dataclass
+class _Costs:
+    substitutions: dict[tuple[str, str], float]
+    insertions: dict[str, float]
+    deletions: dict[str, float]
+
+
+class CostLearner:
+    """
+    Configures and executes the process of learning Levenshtein costs from data.
+
+    This class uses a builder pattern, allowing chaining configuration methods
+    before running the final calculation with .fit().
+
+    Example::
+
+        from ocr_stringdist import CostLearner
+
+        data = [
+            ("Hell0", "Hello"),
+        ]
+        learner = CostLearner().with_smoothing(1.0)
+        wl = learner.fit(data) # Substitution 0 -> o learned with cost < 1.0
+    """
+
+    # Configuration parameters
+    _smoothing_k: float
+
+    # These attributes are set during fitting
+    counts: Optional[TallyCounts] = None
+    vocab_size: Optional[int] = None
+
+    def __init__(self) -> None:
+        self._smoothing_k = 1.0
+
+    def with_smoothing(self, k: float) -> "CostLearner":
+        r"""
+        Sets the smoothing parameter `k`.
+
+        This parameter controls how strongly the model defaults to a uniform
+        probability distribution by adding a "pseudo-count" of `k` to every
+        possible event.
+
+        :param k: The smoothing factor, which must be a non-negative number.
+        :return: The CostLearner instance for method chaining.
+        :raises ValueError: If k < 0.
+
+        Notes
+        -----
+        This parameter allows for a continuous transition between two modes:
+
+        - **k > 0 (recommended):** This enables additive smoothing, with `k = 1.0`
+          being Laplace smoothing. It regularizes the model by assuming no event is impossible.
+          The final costs are a measure of "relative surprisal," normalized by the vocabulary size
+
+        - **k = 0:** This corresponds to a normalized Maximum Likelihood Estimation.
+          Probabilities are derived from the raw observed frequencies. The final costs are
+          normalized using the same logic as the `k > 0` case, making `k=0` the continuous limit
+          of the smoothed model. In this mode, costs can only be calculated for events observed in
+          the training data. Unseen events will receive the default cost, regardless of
+          the value of `calculate_for_unseen` in :meth:`fit`.
+        """
+        if k < 0:
+            raise ValueError("Smoothing parameter k must be non-negative.")
+        self._smoothing_k = k
+        return self
+
+    def _tally_operations(self, operations: Iterable["EditOperation"]) -> TallyCounts:
+        """Tally all edit operations."""
+        counts = TallyCounts()
+        for op in operations:
+            if op.source_token is not None:
+                counts.vocab.add(op.source_token)
+            if op.target_token is not None:
+                counts.target_chars[op.target_token] += 1
+                counts.vocab.add(op.target_token)
+
+            if op.op_type == "substitute":
+                if op.source_token is None or op.target_token is None:
+                    raise ValueError("Tokens cannot be None for 'substitute'")
+                counts.substitutions[(op.source_token, op.target_token)] += 1
+                counts.source_chars[op.source_token] += 1
+            elif op.op_type == "delete":
+                if op.source_token is None:
+                    raise ValueError("Source token cannot be None for 'delete'")
+                counts.deletions[op.source_token] += 1
+                counts.source_chars[op.source_token] += 1
+            elif op.op_type == "insert":
+                if op.target_token is None:
+                    raise ValueError("Target token cannot be None for 'insert'")
+                counts.insertions[op.target_token] += 1
+            elif op.op_type == "match":
+                if op.source_token is None:
+                    raise ValueError("Source token cannot be None for 'match'")
+                counts.source_chars[op.source_token] += 1
+        return counts
+
+    def _calculate_costs(
+        self, counts: TallyCounts, vocab: set[str], calculate_for_unseen: bool = False
+    ) -> _Costs:
+        """
+        Calculates the costs for edit operations based on tallied counts.
+        """
+        sub_costs: dict[tuple[str, str], float] = {}
+        ins_costs: dict[str, float] = {}
+        del_costs: dict[str, float] = {}
+        k = self._smoothing_k
+
+        if k == 0:
+            calculate_for_unseen = False
+
+        # Error space size V for all conditional probabilities.
+        # The space of possible outcomes for a given source character (from OCR)
+        # includes all vocab characters (for matches/substitutions) plus the empty
+        # character (for deletions). This gives V = len(vocab) + 1.
+        # Symmetrically, the space of outcomes for a given target character (from GT)
+        # includes all vocab characters plus the empty character (for insertions/misses).
+        V = len(vocab) + 1
+
+        # Normalization ceiling Z' = -log(1/V).
+        normalization_ceiling = math.log(V) if V > 1 else 1.0
+
+        # Substitutions
+        sub_iterator = (
+            itertools.product(vocab, vocab) if calculate_for_unseen else counts.substitutions.keys()
+        )
+        for source, target in sub_iterator:
+            count = counts.substitutions[(source, target)]
+            total_count = counts.source_chars[source]
+            prob = (count + k) / (total_count + k * V)
+            base_cost = negative_log_likelihood(prob)
+            sub_costs[(source, target)] = base_cost / normalization_ceiling
+
+        # Deletions
+        del_iterator = vocab if calculate_for_unseen else counts.deletions.keys()
+        for source in del_iterator:
+            count = counts.deletions[source]
+            total_count = counts.source_chars[source]
+            prob = (count + k) / (total_count + k * V)
+            base_cost = negative_log_likelihood(prob)
+            del_costs[source] = base_cost / normalization_ceiling
+
+        # Insertions
+        ins_iterator = vocab if calculate_for_unseen else counts.insertions.keys()
+        for target in ins_iterator:
+            count = counts.insertions[target]
+            total_target_count = counts.target_chars[target]
+            prob = (count + k) / (total_target_count + k * V)
+            base_cost = negative_log_likelihood(prob)
+            ins_costs[target] = base_cost / normalization_ceiling
+
+        return _Costs(substitutions=sub_costs, insertions=ins_costs, deletions=del_costs)
+
+    def _calculate_operations(
+        self, pairs: Iterable[tuple[str, str]], aligner: "Aligner"
+    ) -> list["EditOperation"]:
+        """Calculate edit operations for all string pairs using the provided aligner."""
+
+        all_ops = [
+            op
+            for ocr_str, truth_str in pairs
+            for op in aligner.explain(ocr_str, truth_str, filter_matches=False)
+        ]
+        return all_ops
+
+    def fit(
+        self,
+        pairs: Iterable[tuple[str, str]],
+        *,
+        initial_model: "Aligner | None" = None,
+        calculate_for_unseen: bool = False,
+    ) -> "WeightedLevenshtein":
+        """
+        Fits the costs of a WeightedLevenshtein instance to the provided data.
+
+        Note that learning multi-character tokens is only supported if an initial alignment model
+        is provided that can handle those multi-character tokens.
+
+        This method analyzes pairs of strings to learn the costs of edit operations
+        based on their observed frequencies. The underlying model calculates costs
+        based on the principle of relative information cost.
+
+        For a detailed explanation of the methodology, please see the
+        :doc:`Cost Learning Model <cost_learning_model>` documentation page.
+
+        :param pairs: An iterable of (ocr_string, ground_truth_string) tuples.
+        :param initial_model: Optional initial model used to align OCR outputs and ground truth
+                              strings. By default, an unweighted Levenshtein distance is used.
+        :param calculate_for_unseen: If True (and k > 0), pre-calculates costs for all
+                                     possible edit operations based on the vocabulary.
+                                     If False (default), only calculates costs for operations
+                                     observed in the data.
+        :return: A `WeightedLevenshtein` instance with the learned costs.
+        """
+        from .levenshtein import WeightedLevenshtein
+
+        if not pairs:
+            return WeightedLevenshtein.unweighted()
+
+        if initial_model is None:
+            initial_model = WeightedLevenshtein.unweighted()
+
+        all_ops = self._calculate_operations(pairs, aligner=initial_model)
+        self.counts = self._tally_operations(all_ops)
+        vocab = self.counts.vocab
+        self.vocab_size = len(vocab)
+
+        if not self.vocab_size:
+            return WeightedLevenshtein.unweighted()
+
+        costs = self._calculate_costs(self.counts, vocab, calculate_for_unseen=calculate_for_unseen)
+
+        return WeightedLevenshtein(
+            substitution_costs=costs.substitutions,
+            insertion_costs=costs.insertions,
+            deletion_costs=costs.deletions,
+            symmetric_substitution=False,
+            default_substitution_cost=1.0,
+            default_insertion_cost=1.0,
+            default_deletion_cost=1.0,
+        )

ocr_stringdist/levenshtein.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
-from dataclasses import dataclass
-from typing import Literal, Optional
+from collections.abc import Iterable
+from typing import Any, Optional
 
 from ._rust_stringdist import (
     _batch_weighted_levenshtein_distance,
@@ -9,20 +9,7 @@ from ._rust_stringdist import (
     _weighted_levenshtein_distance,
 )
 from .default_ocr_distances import ocr_distance_map
-
-OperationType = Literal["substitute", "insert", "delete"]
-
-
-@dataclass(frozen=True)
-class EditOperation:
-    """
-    Represents a single edit operation (substitution, insertion, or deletion).
-    """
-
-    op_type: OperationType
-    source_token: Optional[str]
-    target_token: Optional[str]
-    cost: float
+from .edit_operation import EditOperation
 
 
 class WeightedLevenshtein:
@@ -33,14 +20,17 @@ class WeightedLevenshtein:
     how the distance is measured. Once created, its methods can be used to
     efficiently compute distances and explain the edit operations.
 
-    :param substitution_costs: Maps (char, char) tuples to their substitution cost.
+    :param substitution_costs: Maps (str, str) tuples to their substitution cost.
                                Defaults to costs based on common OCR errors.
-    :param insertion_costs: Maps a character to its insertion cost.
-    :param deletion_costs: Maps a character to its deletion cost.
-    :param symmetric_substitution: If True, substitution costs are bidirectional.
-    :param default_substitution_cost: Default cost for substitutions not in the map.
-    :param default_insertion_cost: Default cost for insertions not in the map.
-    :param default_deletion_cost: Default cost for deletions not in the map.
+    :param insertion_costs: Maps a string to its insertion cost.
+    :param deletion_costs: Maps a string to its deletion cost.
+    :param symmetric_substitution: If True, a cost defined for, e.g., ('0', 'O') will automatically
+                                   apply to ('O', '0'). If False, both must be defined explicitly.
+    :param default_substitution_cost: Default cost for single-char substitutions not in the map.
+    :param default_insertion_cost: Default cost for single-char insertions not in the map.
+    :param default_deletion_cost: Default cost for single-char deletions not in the map.
+
+    :raises TypeError, ValueError: If the provided arguments are invalid.
     """
 
     substitution_costs: dict[tuple[str, str], float]
@@ -62,9 +52,37 @@ class WeightedLevenshtein:
         default_insertion_cost: float = 1.0,
         default_deletion_cost: float = 1.0,
     ) -> None:
-        self.substitution_costs = (
-            ocr_distance_map if substitution_costs is None else substitution_costs
-        )
+        # Validate default costs
+        for cost_name, cost_val in [
+            ("default_substitution_cost", default_substitution_cost),
+            ("default_insertion_cost", default_insertion_cost),
+            ("default_deletion_cost", default_deletion_cost),
+        ]:
+            if not isinstance(cost_val, (int, float)):
+                raise TypeError(f"{cost_name} must be a number, but got: {type(cost_val).__name__}")
+            if cost_val < 0:
+                raise ValueError(f"{cost_name} must be non-negative, got value: {cost_val}")
+
+        # Validate substitution_costs dictionary
+        sub_costs = ocr_distance_map if substitution_costs is None else substitution_costs
+        for key, cost in sub_costs.items():
+            if not (
+                isinstance(key, tuple)
+                and len(key) == 2
+                and isinstance(key[0], str)
+                and isinstance(key[1], str)
+            ):
+                raise TypeError(
+                    f"substitution_costs keys must be tuples of two strings, but found: {key}"
+                )
+            if not isinstance(cost, (int, float)):
+                raise TypeError(
+                    f"Cost for substitution {key} must be a number, but got: {type(cost).__name__}"
+                )
+            if cost < 0:
+                raise ValueError(f"Cost for substitution {key} cannot be negative, but got: {cost}")
+
+        self.substitution_costs = sub_costs
         self.insertion_costs = {} if insertion_costs is None else insertion_costs
         self.deletion_costs = {} if deletion_costs is None else deletion_costs
         self.symmetric_substitution = symmetric_substitution
@@ -81,162 +99,117 @@ class WeightedLevenshtein:
         """Calculates the weighted Levenshtein distance between two strings."""
         return _weighted_levenshtein_distance(s1, s2, **self.__dict__)  # type: ignore[no-any-return]
 
-    def explain(self, s1: str, s2: str) -> list[EditOperation]:
-        """Returns the list of edit operations to transform s1 into s2."""
+    def explain(self, s1: str, s2: str, filter_matches: bool = True) -> list[EditOperation]:
+        """
+        Returns the list of edit operations to transform s1 into s2.
+
+        :param s1: First string (interpreted as the string read via OCR)
+        :param s2: Second string (interpreted as the target string)
+        :param filter_matches: If True, 'match' operations are excluded from the result.
+        :return: List of :class:`EditOperation` instances.
+        """
         raw_path = _explain_weighted_levenshtein_distance(s1, s2, **self.__dict__)
-        return [EditOperation(*op) for op in raw_path]
+        parsed_path = [EditOperation(*op) for op in raw_path]
+        if filter_matches:
+            return list(filter(lambda op: op.op_type != "match", parsed_path))
+        return parsed_path
 
     def batch_distance(self, s: str, candidates: list[str]) -> list[float]:
         """Calculates distances between a string and a list of candidates."""
         return _batch_weighted_levenshtein_distance(s, candidates, **self.__dict__)  # type: ignore[no-any-return]
 
+    @classmethod
+    def learn_from(cls, pairs: Iterable[tuple[str, str]]) -> WeightedLevenshtein:
+        """
+        Creates an instance by learning costs from a dataset of (OCR, ground truth) string pairs.
+
+        For more advanced learning configuration, see the
+        :class:`ocr_stringdist.learner.CostLearner` class.
+
+        :param pairs: An iterable of (ocr_string, ground_truth_string) tuples. Correct pairs
+                      are not intended to be filtered; they are needed to learn well-aligned costs.
+        :return: A new `WeightedLevenshtein` instance with the learned costs.
+
+        Example::
+
+            from ocr_stringdist import WeightedLevenshtein
+
+            training_data = [
+                ("8N234", "BN234"), # read '8' instead of 'B'
+                ("BJK18", "BJK18"), # correct
+                ("ABC0.", "ABC0"),  # extra '.'
+            ]
+            wl = WeightedLevenshtein.learn_from(training_data)
+            print(wl.substitution_costs) # learned cost for substituting '8' with 'B'
+            print(wl.deletion_costs) # learned cost for deleting '.'
+        """
+        from .learner import CostLearner
+
+        return CostLearner().fit(pairs)
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, WeightedLevenshtein):
+            return NotImplemented
+        return (
+            self.substitution_costs == other.substitution_costs
+            and self.insertion_costs == other.insertion_costs
+            and self.deletion_costs == other.deletion_costs
+            and self.symmetric_substitution == other.symmetric_substitution
+            and self.default_substitution_cost == other.default_substitution_cost
+            and self.default_insertion_cost == other.default_insertion_cost
+            and self.default_deletion_cost == other.default_deletion_cost
+        )
 
-def weighted_levenshtein_distance(
-    s1: str,
-    s2: str,
-    /,
-    substitution_costs: Optional[dict[tuple[str, str], float]] = None,
-    insertion_costs: Optional[dict[str, float]] = None,
-    deletion_costs: Optional[dict[str, float]] = None,
-    *,
-    symmetric_substitution: bool = True,
-    default_substitution_cost: float = 1.0,
-    default_insertion_cost: float = 1.0,
-    default_deletion_cost: float = 1.0,
-) -> float:
-    """
-    Levenshtein distance with custom substitution, insertion and deletion costs.
-
-    See also :meth:`WeightedLevenshtein.distance`.
-
-    The default `substitution_costs` considers common OCR errors, see
-    :py:data:`ocr_stringdist.default_ocr_distances.ocr_distance_map`.
-
-    :param s1: First string (interpreted as the string read via OCR)
-    :param s2: Second string
-    :param substitution_costs: Dictionary mapping tuples of strings ("substitution tokens") to their
-                     substitution costs. Only one direction needs to be configured unless
-                     `symmetric_substitution` is False.
-                     Note that the runtime scales in the length of the longest substitution token.
-                     Defaults to `ocr_stringdist.ocr_distance_map`.
-    :param insertion_costs: Dictionary mapping strings to their insertion costs.
-    :param deletion_costs: Dictionary mapping strings to their deletion costs.
-    :param symmetric_substitution: Should the keys of `substitution_costs` be considered to be
-                                   symmetric? Defaults to True.
-    :param default_substitution_cost: The default substitution cost for character pairs not found
-                                      in `substitution_costs`.
-    :param default_insertion_cost: The default insertion cost for characters not found in
-                                   `insertion_costs`.
-    :param default_deletion_cost: The default deletion cost for characters not found in
-                                  `deletion_costs`.
-    """
-    return WeightedLevenshtein(
-        substitution_costs=substitution_costs,
-        insertion_costs=insertion_costs,
-        deletion_costs=deletion_costs,
-        symmetric_substitution=symmetric_substitution,
-        default_substitution_cost=default_substitution_cost,
-        default_insertion_cost=default_insertion_cost,
-        default_deletion_cost=default_deletion_cost,
-    ).distance(s1, s2)
-
-
-def batch_weighted_levenshtein_distance(
-    s: str,
-    candidates: list[str],
-    /,
-    substitution_costs: Optional[dict[tuple[str, str], float]] = None,
-    insertion_costs: Optional[dict[str, float]] = None,
-    deletion_costs: Optional[dict[str, float]] = None,
-    *,
-    symmetric_substitution: bool = True,
-    default_substitution_cost: float = 1.0,
-    default_insertion_cost: float = 1.0,
-    default_deletion_cost: float = 1.0,
-) -> list[float]:
-    """
-    Calculate weighted Levenshtein distances between a string and multiple candidates.
-
-    See also :meth:`WeightedLevenshtein.batch_distance`.
-
-    This is more efficient than calling :func:`weighted_levenshtein_distance` multiple times.
-
-    :param s: The string to compare (interpreted as the string read via OCR)
-    :param candidates: List of candidate strings to compare against
-    :param substitution_costs: Dictionary mapping tuples of strings ("substitution tokens") to their
-                     substitution costs. Only one direction needs to be configured unless
-                     `symmetric_substitution` is False.
-                     Note that the runtime scales in the length of the longest substitution token.
-                     Defaults to `ocr_stringdist.ocr_distance_map`.
-    :param insertion_costs: Dictionary mapping strings to their insertion costs.
-    :param deletion_costs: Dictionary mapping strings to their deletion costs.
-    :param symmetric_substitution: Should the keys of `substitution_costs` be considered to be
-                                   symmetric? Defaults to True.
-    :param default_substitution_cost: The default substitution cost for character pairs not found
-                                      in `substitution_costs`.
-    :param default_insertion_cost: The default insertion cost for characters not found in
-                                   `insertion_costs`.
-    :param default_deletion_cost: The default deletion cost for characters not found in
-                                  `deletion_costs`.
-    :return: A list of distances corresponding to each candidate
-    """
-    return WeightedLevenshtein(
-        substitution_costs=substitution_costs,
-        insertion_costs=insertion_costs,
-        deletion_costs=deletion_costs,
-        symmetric_substitution=symmetric_substitution,
-        default_substitution_cost=default_substitution_cost,
-        default_insertion_cost=default_insertion_cost,
-        default_deletion_cost=default_deletion_cost,
-    ).batch_distance(s, candidates)
-
-
-def explain_weighted_levenshtein(
-    s1: str,
-    s2: str,
-    /,
-    substitution_costs: Optional[dict[tuple[str, str], float]] = None,
-    insertion_costs: Optional[dict[str, float]] = None,
-    deletion_costs: Optional[dict[str, float]] = None,
-    *,
-    symmetric_substitution: bool = True,
-    default_substitution_cost: float = 1.0,
-    default_insertion_cost: float = 1.0,
-    default_deletion_cost: float = 1.0,
-) -> list[EditOperation]:
-    """
-    Computes the path of operations associated with the custom Levenshtein distance.
-
-    See also :meth:`WeightedLevenshtein.explain`.
-
-    The default `substitution_costs` considers common OCR errors, see
-    :py:data:`ocr_stringdist.default_ocr_distances.ocr_distance_map`.
-
-    :param s1: First string (interpreted as the string read via OCR)
-    :param s2: Second string
-    :param substitution_costs: Dictionary mapping tuples of strings ("substitution tokens") to their
-                     substitution costs. Only one direction needs to be configured unless
-                     `symmetric_substitution` is False.
-                     Note that the runtime scales in the length of the longest substitution token.
-                     Defaults to `ocr_stringdist.ocr_distance_map`.
-    :param insertion_costs: Dictionary mapping strings to their insertion costs.
-    :param deletion_costs: Dictionary mapping strings to their deletion costs.
-    :param symmetric_substitution: Should the keys of `substitution_costs` be considered to be
-                                   symmetric? Defaults to True.
-    :param default_substitution_cost: The default substitution cost for character pairs not found
-                                      in `substitution_costs`.
-    :param default_insertion_cost: The default insertion cost for characters not found in
-                                   `insertion_costs`.
-    :param default_deletion_cost: The default deletion cost for characters not found in
-                                  `deletion_costs`.
-    :return: List of :class:`EditOperation` instances.
-    """
-    return WeightedLevenshtein(
-        substitution_costs=substitution_costs,
-        insertion_costs=insertion_costs,
-        deletion_costs=deletion_costs,
-        symmetric_substitution=symmetric_substitution,
-        default_substitution_cost=default_substitution_cost,
-        default_insertion_cost=default_insertion_cost,
-        default_deletion_cost=default_deletion_cost,
-    ).explain(s1, s2)
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the instance's configuration to a dictionary.
+
+        The result can be written to, say, JSON.
+
+        For the counterpart, see :meth:`WeightedLevenshtein.from_dict`.
+        """
+        # Convert tuple keys to a list of lists/objects for broader compatibility (e.g., JSON)
+        sub_costs_serializable = [
+            {"from": k[0], "to": k[1], "cost": v} for k, v in self.substitution_costs.items()
+        ]
+
+        return {
+            "substitution_costs": sub_costs_serializable,
+            "insertion_costs": self.insertion_costs,
+            "deletion_costs": self.deletion_costs,
+            "symmetric_substitution": self.symmetric_substitution,
+            "default_substitution_cost": self.default_substitution_cost,
+            "default_insertion_cost": self.default_insertion_cost,
+            "default_deletion_cost": self.default_deletion_cost,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> WeightedLevenshtein:
+        """
+        Deserialize from a dictionary.
+
+        For the counterpart, see :meth:`WeightedLevenshtein.to_dict`.
+
+        :param data: A dictionary with (not necessarily all of) the following keys:
+                     - "substitution_costs": {"from": str, "to": str, "cost": float}
+                     - "substitution_costs": dict[str, float]
+                     - "deletion_costs": dict[str, float]
+                     - "symmetric_substitution": bool
+                     - "default_substitution_cost": float
+                     - "default_insertion_cost": float
+                     - "default_deletion_cost": float
+        """
+        # Convert the list of substitution costs back to the required dict format
+        sub_costs: dict[tuple[str, str], float] = {
+            (item["from"], item["to"]): item["cost"] for item in data.get("substitution_costs", {})
+        }
+
+        return cls(
+            substitution_costs=sub_costs,
+            insertion_costs=data.get("substitution_costs"),
+            deletion_costs=data.get("deletion_costs"),
+            symmetric_substitution=data.get("symmetric_substitution", True),
+            default_substitution_cost=data.get("default_substitution_cost", 1.0),
+            default_insertion_cost=data.get("default_insertion_cost", 1.0),
+            default_deletion_cost=data.get("default_deletion_cost", 1.0),
+        )

ocr_stringdist/matching.py

@@ -39,13 +39,13 @@ def find_best_candidate(
              calculated distance/score.
     :rtype: tuple[str, float]
 
-    :Example:
+    Example::
 
-    >>> from ocr_stringdist import weighted_levenshtein_distance as distance
-    >>> s = "apple"
-    >>> candidates = ["apply", "apples", "orange", "appIe"]
-    >>> find_best_match(s, candidates, lambda s1, s2: distance(s1, s2, {("l", "I"): 0.1}))
-    ('appIe', 0.1)
+        from ocr_stringdist import find_best_candidate, WeightedLevenshtein
+
+        wl = WeightedLevenshtein({("l", "I"): 0.1})
+        find_best_candidate("apple", ["apply", "apples", "orange", "appIe"], wl.distance)
+        # ('appIe', 0.1)
     """
     if not candidates:
         raise ValueError("The 'candidates' iterable cannot be empty.")

ocr_stringdist/protocols.py

@@ -0,0 +1,9 @@
+from typing import TYPE_CHECKING, Protocol, runtime_checkable
+
+if TYPE_CHECKING:
+    from .edit_operation import EditOperation
+
+
+@runtime_checkable
+class Aligner(Protocol):
+    def explain(self, s1: str, s2: str, filter_matches: bool) -> list["EditOperation"]: ...

ocr_stringdist-1.0.0.dist-info/METADATA

@@ -0,0 +1,94 @@
+Metadata-Version: 2.4
+Name: ocr-stringdist
+Version: 1.0.0
+Classifier: Programming Language :: Rust
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+License-File: LICENSE
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: repository, https://github.com/NiklasvonM/ocr-stringdist
+Project-URL: documentation, https://niklasvonm.github.io/ocr-stringdist/
+
+# OCR-StringDist
+
+A Python library to learn, model, explain and correct OCR errors using a fast string distance engine.
+
+Documentation: https://niklasvonm.github.io/ocr-stringdist/
+
+[![PyPI badge](https://badge.fury.io/py/ocr-stringdist.svg)](https://badge.fury.io/py/ocr-stringdist)
+[![License](https://img.shields.io/badge/License-MIT-green)](LICENSE)
+
+## Overview
+
+Standard string distances (like Levenshtein) treat all character substitutions equally. This is suboptimal for text read from images via OCR, where errors like `O` vs `0` are far more common than, say, `O` vs `X`.
+
+OCR-StringDist provides a learnable **weighted Levenshtein distance**, implementing part of the **Noisy Channel model**.
+
+**Example:** Matching against the correct word `CODE`:
+
+* **Standard Levenshtein:**
+    * $d(\text{"CODE"}, \text{"C0DE"}) = 1$ (O → 0)
+    * $d(\text{"CODE"}, \text{"CXDE"}) = 1$ (O → X)
+    * Result: Both appear equally likely/distant.
+
+* **OCR-StringDist (Channel Model):**
+    * $d(\text{"CODE"}, \text{"C0DE"}) \approx 0.1$ (common error, low cost)
+    * $d(\text{"CODE"}, \text{"CXDE"}) = 1.0$ (unlikely error, high cost)
+    * Result: Correctly identifies `C0DE` as a much closer match.
+
+This makes it ideal for matching potentially incorrect OCR output against known values (e.g., product codes). By combining this *channel model* with a *source model* (e.g., product code frequencies), you can build a complete and robust OCR correction system.
+
+## Installation
+
+```bash
+pip install ocr-stringdist
+```
+
+## Features
+
+- **Learnable Costs**: Automatically learn substitution, insertion, and deletion costs from a dataset of (OCR string, ground truth string) pairs.
+- **Weighted Levenshtein Distance**: Models OCR error patterns by assigning custom costs to specific edit operations.
+- **High Performance**: Core logic in Rust and a batch_distance function for efficiently comparing one string against thousands of candidates.
+- **Substitution of Multiple Characters**: Not just character pairs, but string pairs may be substituted, for example the Korean syllable "이" for the two letters "OI".
+- **Explainable Edit Path**: Returns the optimal sequence of edit operations (substitutions, insertions, and deletions) used to transform one string into another.
+- **Pre-defined OCR Distance Map**: A built-in distance map for common OCR confusions (e.g., "0" vs "O", "1" vs "l", "5" vs "S").
+- **Full Unicode Support**: Works with arbitrary Unicode strings.
+
+## Core Workflow
+
+The typical workflow involves
+- learning costs from your data and then
+- using the resulting model to find the best match from a list of candidates.
+
+```python
+from ocr_stringdist import WeightedLevenshtein
+
+# 1. LEARN costs from your own data
+training_data = [
+    ("128", "123"),
+    ("567", "567"),
+]
+wl = WeightedLevenshtein.learn_from(training_data)
+
+# The engine has now learned that '8' -> '3' is a low-cost substitution
+print(f"Learned cost for ('8', '3'): {wl.substitution_costs[('8', '3')]:.2f}")
+
+
+# 2. MATCH new OCR output against a list of candidates
+ocr_output = "Product Code 128"
+candidates = [
+    "Product Code 123",
+    "Product Code 523",  # '5' -> '1' is an unlikely error
+]
+
+distances = wl.batch_distance(ocr_output, candidates)
+
+# Find the best match
+min_distance = min(distances)
+best_match = candidates[distances.index(min_distance)]
+
+print(f"Best match for '{ocr_output}': '{best_match}' (Cost: {min_distance:.2f})")
+```
+

ocr_stringdist-1.0.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+ocr_stringdist-1.0.0.dist-info/METADATA,sha256=sFZnhhX8kHoYFbMua4zHCq2tELQPXQw3vWGNRoStR-4,3963
+ocr_stringdist-1.0.0.dist-info/WHEEL,sha256=6cglYgN2x9bsL8KQgndplH9dyQf2UyeKi6L__-GPsk0,107
+ocr_stringdist-1.0.0.dist-info/licenses/LICENSE,sha256=5BPRcjlnbl2t4TidSgpfGrtC_birSf8JlZfA-qmVoQE,1072
+ocr_stringdist.libs/libgcc_s-98a1ef30.so.1,sha256=XOVRhHznCIpbSdhFoozhla-OfRqBtXftKPQ4cSMKjrs,433441
+ocr_stringdist/__init__.py,sha256=mL-19TkQQElK5B6iVFCV7vjKVal-6JcsBOFKwiCPQnA,284
+ocr_stringdist/_rust_stringdist.cpython-313-x86_64-linux-musl.so,sha256=ixwa7I8bvHbWfMNN40Wkdsnvn585CdzOC88U9gaM2b4,826057
+ocr_stringdist/default_ocr_distances.py,sha256=oSu-TpHjPA4jxKpLAfmap8z0ZsC99jsOjnRVHW7Hj_Y,1033
+ocr_stringdist/edit_operation.py,sha256=EgEc-2_nOwLUZDOWtogYqKLXIQJxOd9sIAbcGkn-TMY,395
+ocr_stringdist/learner.py,sha256=3qWvqHrAWm4seuwmBmFN4InRL20u8HnPATHjCTnU3I0,10491
+ocr_stringdist/levenshtein.py,sha256=t05FicwL5WTTsRSzDa92v79D2LpDiEUOYG_6te8oT28,9854
+ocr_stringdist/matching.py,sha256=28Xt-x_V_iVsohD3F64MfZ0mys4_qOZXTIAcmSOE0dA,3270
+ocr_stringdist/protocols.py,sha256=IyvGzzktPgmPRZyDRE0UKCYo4C0tdewU8IgwFbxZLls,265
+ocr_stringdist/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ocr_stringdist-1.0.0.dist-info/RECORD,,

ocr_stringdist-0.2.2.dist-info/METADATA

@@ -1,102 +0,0 @@
-Metadata-Version: 2.4
-Name: ocr-stringdist
-Version: 0.2.2
-Classifier: Programming Language :: Rust
-Classifier: Programming Language :: Python
-Classifier: Operating System :: OS Independent
-License-File: LICENSE
-Requires-Python: >=3.9
-Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
-Project-URL: repository, https://github.com/NiklasvonM/ocr-stringdist
-Project-URL: documentation, https://niklasvonm.github.io/ocr-stringdist/
-
-# OCR-StringDist
-
-A Python library for fast string distance calculations that account for common OCR (optical character recognition) errors.
-
-Documentation: https://niklasvonm.github.io/ocr-stringdist/
-
-[![PyPI](https://img.shields.io/badge/PyPI-Package-blue)](https://pypi.org/project/ocr-stringdist/)
-[![License](https://img.shields.io/badge/License-MIT-green)](LICENSE)
-
-## Overview
-
-Standard string distances (like Levenshtein) treat all character substitutions equally. This is suboptimal for text read from images via OCR, where errors like `O` vs `0` are far more common than, say, `O` vs `X`.
-
-OCR-StringDist uses a **weighted Levenshtein distance**, assigning lower costs to common OCR errors.
-
-**Example:** Matching against the correct word `CODE`:
-
-* **Standard Levenshtein:**
-    * $d(\text{"CODE"}, \text{"C0DE"}) = 1$ (O → 0)
-    * $d(\text{"CODE"}, \text{"CXDE"}) = 1$ (O → X)
-    * Result: Both appear equally likely/distant.
-
-* **OCR-StringDist (Weighted):**
-    * $d(\text{"CODE"}, \text{"C0DE"}) \approx 0.1$ (common error, low cost)
-    * $d(\text{"CODE"}, \text{"CXDE"}) = 1.0$ (unlikely error, high cost)
-    * Result: Correctly identifies `C0DE` as a much closer match.
-
-This makes it ideal for matching potentially incorrect OCR output against known values (e.g., product codes, database entries).
-
-## Installation
-
-```bash
-pip install ocr-stringdist
-```
-
-## Features
-
-- **High Performance**: The core logic is implemented in Rust with speed in mind.
-- **Weighted Levenshtein Distance**: Calculates Levenshtein distance with customizable costs for substitutions, insertions, and deletions. Includes an efficient batch version (`batch_weighted_levenshtein_distance`) for comparing one string against many candidates.
-- **Explainable Edit Path**: Returns the optimal sequence of edit operations (substitutions, insertions, and deletions) used to transform one string into another.
-- **Substitution of Multiple Characters**: Not just character pairs, but string pairs may be substituted, for example the Korean syllable "이" for the two letters "OI".
-- **Pre-defined OCR Distance Map**: A built-in distance map for common OCR confusions (e.g., "0" vs "O", "1" vs "l", "5" vs "S").
-- **Unicode Support**: Works with arbitrary Unicode strings.
-- **Best Match Finder**: Includes a utility function `find_best_candidate` to efficiently find the best match from a list based on _any_ distance function.
-
-## Usage
-
-### Basic usage
-
-```python
-from ocr_stringdist import WeightedLevenshtein
-
-# Default substitution costs are ocr_stringdist.ocr_distance_map.
-wl = WeightedLevenshtein()
-
-print(wl.distance("CXDE", "CODE")) # == 1
-print(wl.distance("C0DE", "CODE")) # < 1
-```
-
-### Explain the Edit Path
-
-```python
-edit_path = wl.explain("C0DE", "CODE")
-print(edit_path)
-# EditOperation(op_type='substitute', source_token='0', target_token='O', cost=0.1)]
-```
-
-### Fast Batch Calculations
-
-Quickly compare a string to a list of candidates.
-
-```python
-distances: list[float] = wl.batch_distance("CODE", ["CXDE", "C0DE"])
-# [1.0, 0.1]
-```
-
-### Multi-character Substitutions
-
-```python
-# Custom costs with multi-character substitution
-wl = WeightedLevenshtein(substitution_costs={("In", "h"): 0.5})
-
-print(wl.distance("hi", "Ini")) # 0.5
-```
-
-
-## Acknowledgements
-
-This project is inspired by [jellyfish](https://github.com/jamesturk/jellyfish), providing the base implementations of the algorithms used here.
-

ocr_stringdist-0.2.2.dist-info/RECORD

@@ -1,11 +0,0 @@
-ocr_stringdist-0.2.2.dist-info/METADATA,sha256=2KjG6DHqpsannN0lPK4EwkYBbY3adZrl1oTCq-elnL8,3868
-ocr_stringdist-0.2.2.dist-info/WHEEL,sha256=6cglYgN2x9bsL8KQgndplH9dyQf2UyeKi6L__-GPsk0,107
-ocr_stringdist-0.2.2.dist-info/licenses/LICENSE,sha256=5BPRcjlnbl2t4TidSgpfGrtC_birSf8JlZfA-qmVoQE,1072
-ocr_stringdist.libs/libgcc_s-98a1ef30.so.1,sha256=XOVRhHznCIpbSdhFoozhla-OfRqBtXftKPQ4cSMKjrs,433441
-ocr_stringdist/__init__.py,sha256=ApxqraLRcWAkzXhGJXSf3EqGEVFbxghrYrfJ9dmQjQU,467
-ocr_stringdist/_rust_stringdist.cpython-313-x86_64-linux-musl.so,sha256=EcPdJhjPSI5_vBlDULel2-m6U_5pncK31yxEyRFZMVY,821945
-ocr_stringdist/default_ocr_distances.py,sha256=oSu-TpHjPA4jxKpLAfmap8z0ZsC99jsOjnRVHW7Hj_Y,1033
-ocr_stringdist/levenshtein.py,sha256=Jypg31BQyULipJ_Yh3dcBQDKNnbvEIlmf28tDr_gySw,11243
-ocr_stringdist/matching.py,sha256=rr8R63Ttu2hTf5Mni7_P8aGBbjWs6t2QPV3wxKXspAs,3293
-ocr_stringdist/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ocr_stringdist-0.2.2.dist-info/RECORD,,