resolvekit 0.1.3__tar.gz → 0.1.5__tar.gz

{resolvekit-0.1.3 → resolvekit-0.1.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: resolvekit
-Version: 0.1.3
+Version: 0.1.5
 Summary: Entity and place resolution system that maps messy place/entity strings and codes to canonical entities
 Keywords: entity-resolution,geocoding,place-names,data-commons,iso-codes,offline,disambiguation,normalization
 Author: Jorge Rivera

{resolvekit-0.1.3 → resolvekit-0.1.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "resolvekit"
-version = "0.1.3"
+version = "0.1.5"
 description = "Entity and place resolution system that maps messy place/entity strings and codes to canonical entities"
 requires-python = ">=3.12"
 

{resolvekit-0.1.3 → resolvekit-0.1.5}/src/resolvekit/core/engine/suggest_rank.py

@@ -49,15 +49,26 @@ PROMINENCE_LIVE_TYPE_PREFIXES: Final[frozenset[str]] = frozenset(
         "geo.subregion",
         "geo.region",
         "geo.continental_union",
-        # City and admin2 tiers gain live Wikidata sitelink / DC population
-        # prominence once the full geo build includes those tiers.  The prefix
-        # is added here so ``ranking_quality`` reports ``"ranked"`` as soon as
-        # the data ships, without a follow-on code change.
+        # City and all sub-country admin tiers carry live prominence in the
+        # shipped remote packs (Wikidata sitelinks with DC population
+        # fallback for admin3/admin4).
         "geo.city",
+        "geo.admin1",
         "geo.admin2",
+        "geo.admin3",
+        "geo.admin4",
+        "geo.admin5",
     }
 )
 
+# Prominence floor for the exact-match lift on prominence-ranked tiers. An
+# exact full-name hit from a ranked tier below this floor loses its lift: an
+# obscure place whose name merely coincides with the prefix ("Germ", a French
+# commune) must not outrank a famous completion ("Germany"). Unranked tiers
+# (orgs — no prominence data) always keep the lift, so typed acronyms ("UN",
+# "NATO") still surface first.
+EXACT_MATCH_MIN_PROMINENCE: Final[float] = 0.3
+
 # Numeric rank for each MatchClass (lower = better).
 MATCH_CLASS_RANK: Final[dict[MatchClass, int]] = {
     MatchClass.EXACT_PREFIX: 0,
@@ -143,15 +154,22 @@ def sort_key(
     ``exact_match_rank`` is 0 when the user's query equals the matched name in
     full (``exact_match=True``) and 1 otherwise.  This lifts entities whose
     complete short name was typed (e.g. "EU", "NATO") above longer-named
-    entities that merely *start with* those letters, even when those entities
-    carry higher prominence scores.
+    entities that merely *start with* those letters — with one exception:
+    a candidate from a prominence-ranked tier whose prominence falls below
+    ``EXACT_MATCH_MIN_PROMINENCE`` loses the lift, so an obscure place whose
+    full name coincides with the prefix ("Germ") cannot outrank a famous
+    completion ("Germany"). Unranked tiers always keep the lift.
 
     The final ``entity_id`` field gives a total order so that equal inputs
     always produce the same sequence.
     """
+    exact_lift = c.exact_match and (
+        ranking_quality(c.entity_type) == "unranked"
+        or c.prominence >= EXACT_MATCH_MIN_PROMINENCE
+    )
     return (
         MATCH_CLASS_RANK[c.match_class],
-        0 if c.exact_match else 1,
+        0 if exact_lift else 1,
         c.typo_count,
         -c.prominence,
         c.name_kind_rank,
@@ -218,6 +236,7 @@ def fuzzy_candidates(
     names: list[tuple[str, str, str, bool, str]],
     *,
     top_k: int,
+    choices: list[str] | None = None,
 ) -> list[SuggestCandidate]:
     """Run brute-force RapidFuzz over a pre-materialized name list.
 
@@ -234,6 +253,12 @@ def fuzzy_candidates(
             value)`` 5-tuples — typically the memoized output of
             ``store.iter_suggest_names()``.
         top_k: Number of results the caller ultimately wants.
+        choices: Optional pre-extracted list of ``value_norm`` strings
+            (``[row[0] for row in names]``).  When provided, the per-call
+            list comprehension is skipped — callers that memoize the name list
+            should also memoize and pass ``choices`` to avoid a 25k-string
+            rebuild on every query.  When ``None`` (default), the list is
+            built from ``names`` as before (backward-compatible).
 
     Returns:
         List of ``SuggestCandidate`` objects (one per distinct matched name row,
@@ -250,8 +275,9 @@ def fuzzy_candidates(
 
     fuzzy_pool = min(max(top_k * 5, 50), 500)
 
-    # Extract just the value_norm strings for RapidFuzz.
-    choices = [row[0] for row in names]
+    # Use caller-supplied choices when available to avoid a per-call rebuild.
+    if choices is None:
+        choices = [row[0] for row in names]
 
     raw_hits = process.extract(
         query_norm,

{resolvekit-0.1.3 → resolvekit-0.1.5}/src/resolvekit/packs/custom/pack.py

@@ -21,6 +21,7 @@ from resolvekit.packs.custom.sources import (
     CustomExactCodeSource,
     CustomExactNameSource,
     CustomFTSSource,
+    CustomFuzzyRetrievalSource,
     CustomFuzzySource,
 )
 
@@ -44,13 +45,22 @@ class GenericPack:
     custom entities, etc.).
 
     Sources (in pipeline order):
-    1. CustomExactCodeSource  — catch-all ``lookup_code_any``; raw 1.0
-    2. CustomExactNameSource  — canonical raw 1.0, alias raw 0.95
-    3. CustomFTSSource        — BM25 ranked FTS
-    4. CustomFuzzySource      — reranks existing candidates (requires_existing_candidates)
-
-    No SymSpell source: custom packs are built programmatically; the fuzzy
-    reranking over FTS/exact candidates serves as the fuzzy story.
+    1. CustomExactCodeSource        — catch-all ``lookup_code_any``; raw 1.0
+    2. CustomExactNameSource        — canonical raw 1.0, alias raw 0.95
+    3. CustomFTSSource              — BM25 ranked FTS
+    4. CustomFuzzyRetrievalSource   — generating brute-force RapidFuzz over the
+                                      store's materialized name list; emits
+                                      FUZZY-tier evidence so typo'd queries that
+                                      FTS cannot tokenize-match still produce
+                                      candidates.  Free on exact-name queries
+                                      (engine fuzzy-skip guard bypasses it when
+                                      a confident EXACT_NAME candidate is present).
+                                      Callers needing stricter precision can raise
+                                      ``confidence_threshold`` above the 0.89 FUZZY
+                                      cap to suppress fuzzy-tier results.
+    5. CustomFuzzySource            — reranks existing candidates with
+                                      ``fuzzy_edit_sim`` / ``fuzzy_token_sim``
+                                      signals (requires_existing_candidates)
 
     Args:
         symspell_dict_path: Accepted but ignored to satisfy factory introspection.
@@ -70,6 +80,7 @@ class GenericPack:
             CustomExactCodeSource(),
             CustomExactNameSource(),
             CustomFTSSource(),
+            CustomFuzzyRetrievalSource(),
             CustomFuzzySource(),
         ]
 

{resolvekit-0.1.3 → resolvekit-0.1.5}/src/resolvekit/packs/custom/sources/__init__.py

@@ -4,10 +4,12 @@ from resolvekit.packs.custom.sources.exact_code import CustomExactCodeSource
 from resolvekit.packs.custom.sources.exact_name import CustomExactNameSource
 from resolvekit.packs.custom.sources.fts import CustomFTSSource
 from resolvekit.packs.custom.sources.fuzzy import CustomFuzzySource
+from resolvekit.packs.custom.sources.fuzzy_retrieval import CustomFuzzyRetrievalSource
 
 __all__ = [
     "CustomExactCodeSource",
     "CustomExactNameSource",
     "CustomFTSSource",
+    "CustomFuzzyRetrievalSource",
     "CustomFuzzySource",
 ]

resolvekit-0.1.5/src/resolvekit/packs/custom/sources/fuzzy_retrieval.py

@@ -0,0 +1,17 @@
+"""Brute-force fuzzy retrieval source for custom entities."""
+
+from resolvekit.shared.sources import FuzzyRetrievalBruteSource
+
+
+class CustomFuzzyRetrievalSource(FuzzyRetrievalBruteSource):
+    """Generating fuzzy source for the custom domain.
+
+    Materializes the store's name list once and runs brute-force
+    RapidFuzz over it — no prebuilt SymSpell dictionary required.
+    Ordered before the rerank ``CustomFuzzySource`` so the engine
+    enriches its FUZZY-tier candidates with ``fuzzy_edit_sim`` /
+    ``fuzzy_token_sim`` signals before scoring.
+    """
+
+    def __init__(self) -> None:
+        super().__init__(name="custom_fuzzy_retrieval", domain="custom")

{resolvekit-0.1.3 → resolvekit-0.1.5}/src/resolvekit/shared/sources/__init__.py

@@ -4,11 +4,15 @@ from resolvekit.shared.sources.code_helpers import evidence_from_code_hits
 from resolvekit.shared.sources.fts_base import BM25ScoreTiers, FTSSource
 from resolvekit.shared.sources.fuzzy_base import FuzzySource
 from resolvekit.shared.sources.fuzzy_retrieval_base import FuzzyRetrievalSource
+from resolvekit.shared.sources.fuzzy_retrieval_brute_base import (
+    FuzzyRetrievalBruteSource,
+)
 from resolvekit.shared.sources.symspell_base import SymSpellSource
 
 __all__ = [
     "BM25ScoreTiers",
     "FTSSource",
+    "FuzzyRetrievalBruteSource",
     "FuzzyRetrievalSource",
     "FuzzySource",
     "SymSpellSource",

resolvekit-0.1.5/src/resolvekit/shared/sources/fuzzy_retrieval_brute_base.py

@@ -0,0 +1,403 @@
+"""Store-backed brute-force fuzzy retrieval source.
+
+Retrieves candidates by running RapidFuzz ``partial_ratio`` over the store's
+materialized name list, reusing the same ``fuzzy_candidates`` / ``iter_suggest_names``
+mechanism the ``suggest()`` path already uses.  Unlike the SymSpell-backed
+``FuzzyRetrievalSource``, this source needs no prebuilt dictionary — it operates
+directly over whatever names are in the store, making it suitable for
+programmatically-built packs (``Resolver.from_records``, ``domain="custom"``).
+
+The engine's fuzzy-skip guard (``should_skip_source``) bypasses this source
+when a confident exact-name match is already present, so the always-on
+registration is free on clean canonical-name queries.
+
+Short-input guard
+-----------------
+The self-contained guard helper at the bottom of this module rejects degenerate
+inputs before the name list is built.  It is intentionally domain-agnostic:
+no geo entity-type hint logic, no ISO-code unlocking.  Add domain-specific
+unlocking in a subclass override if needed.
+"""
+
+from __future__ import annotations
+
+import logging
+import threading
+import time
+from typing import Any
+
+from resolvekit.core.engine import CandidateSource
+from resolvekit.core.engine.suggest_rank import FUZZY_AUTO_MAX_NAMES, fuzzy_candidates
+from resolvekit.core.explain import emit_candidates_generated
+from resolvekit.core.model import (
+    CandidateEvidence,
+    GenerationContext,
+    MatchTier,
+    ReasonCode,
+)
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Short-input guard (domain-agnostic)
+# ---------------------------------------------------------------------------
+
+# Common spreadsheet / dataframe missing-value markers.  Compared casefolded.
+_DEGENERATE_TOKENS: frozenset[str] = frozenset(
+    {
+        "na",
+        "n/a",
+        "n.a.",
+        "n.a",
+        "n/k",
+        "null",
+        "none",
+        "nan",
+        "nil",
+        "tbd",
+        "tba",
+        "unknown",
+        "<null>",
+        "#n/a",
+        "?",
+        "-",
+        "--",
+        "---",
+        ".",
+    }
+)
+
+
+def _is_degenerate_token(text: str) -> bool:
+    """Return True for known missing-value markers (``NA``, ``#N/A``, ``--`` …).
+
+    The check is casefolded; surrounding ASCII whitespace is stripped first.
+    """
+    return text.strip().casefold() in _DEGENERATE_TOKENS
+
+
+def _is_punctuation_noise(text: str) -> bool:
+    """Return True for short tokens dominated by punctuation or symbols.
+
+    Strips common spreadsheet punctuation and checks whether the residual
+    is empty or a very short alpha fragment (≤ 3 chars).  Does not need a
+    dotted-initialism exemption because those pass the ``min_query_length``
+    gate upstream (``U.S.A.`` is 5 chars with dots).
+    """
+    if not text:
+        return True
+    stripped = text
+    for ch in "#/\\-_.,;:!?*|()[]{}'\"`":
+        stripped = stripped.replace(ch, "")
+    stripped = stripped.strip()
+    if not stripped:
+        return True
+    had_punctuation = stripped != text.strip()
+    if not had_punctuation:
+        return False
+    return len(stripped) <= 3 and stripped.isalpha()
+
+
+def _is_single_letter(text: str) -> bool:
+    """Return True for a bare single ASCII letter (any case)."""
+    raw = text.strip()
+    return len(raw) == 1 and raw.isascii() and raw.isalpha()
+
+
+def _short_input_blocked(query_norm: str) -> bool:
+    """Domain-agnostic short-input gate.
+
+    Returns True when the source should suppress itself for this query.
+
+    Checks (earlier wins):
+      1. Degenerate missing-value markers — always blocked.
+      2. Single ASCII letter (any case) — too ambiguous for brute-force fuzzy.
+      3. Punctuation-noise tokens.
+
+    ``min_query_length`` is checked separately in ``generate()`` before this
+    helper is called so callers that relax the default minimum still get the
+    degenerate-token and single-letter blocks.
+    """
+    if _is_degenerate_token(query_norm):
+        return True
+    if _is_single_letter(query_norm):
+        return True
+    return _is_punctuation_noise(query_norm)
+
+
+# ---------------------------------------------------------------------------
+# Source
+# ---------------------------------------------------------------------------
+
+
+class FuzzyRetrievalBruteSource(CandidateSource):
+    """Store-backed generating fuzzy source using brute-force RapidFuzz.
+
+    Materializes the store's name list once, lazily on the first
+    ``generate()`` call (``warm()`` cannot pre-build it because
+    ``CandidateSource.warm()`` receives no store reference), then runs
+    ``fuzzy_candidates`` over it on every query that is not short-circuited by
+    the short-input guard or the engine's skip logic.
+
+    ``tier = MatchTier.FUZZY`` is declared as a **class attribute** so that
+    ``should_skip_source`` (``core/engine/_stages.py``) can read it before
+    ``generate()`` is called — this is the mechanism that makes the always-on
+    registration free on exact-name queries.
+
+    The name list is keyed by the fixed ``entity_type_prefixes`` passed at
+    construction time.  For a custom pack (``entity_type_prefixes=None``) there
+    is exactly one cache entry per instance.  The store is treated as immutable
+    for the resolver's lifetime, so no invalidation is performed.
+
+    Both ``_names_cache`` and ``_choices_cache`` (the extracted ``value_norm``
+    strings) are memoized to avoid rebuilding the ``choices`` list on every
+    call to ``fuzzy_candidates``.
+
+    Configurable parameters
+    -----------------------
+    name : str
+        Unique name for this source (e.g. ``"custom_fuzzy_retrieval"``).
+    domain : str
+        Domain pack ID this source supports (e.g. ``"custom"``).
+    min_query_length : int
+        Minimum normalized query length to process.  Shorter queries return
+        ``[]`` immediately.  Default 3.
+    max_names : int
+        Cap on the name-list size.  When ``len(names) > max_names`` the
+        brute-force pass is skipped and ``[]`` is returned.  Default
+        ``FUZZY_AUTO_MAX_NAMES`` (25 000).
+    top_k : int
+        Maximum candidates requested from ``fuzzy_candidates``.  Default 25.
+    entity_type_prefixes : frozenset[str] | None
+        When given, only names whose entity type starts with one of these
+        prefixes are included in the name list.  ``None`` = all types.
+    """
+
+    # Class-level declaration — read by should_skip_source BEFORE generate() runs.
+    tier: MatchTier = MatchTier.FUZZY  # type: ignore[assignment]
+
+    def __init__(
+        self,
+        *,
+        name: str,
+        domain: str,
+        min_query_length: int = 3,
+        max_names: int = FUZZY_AUTO_MAX_NAMES,
+        top_k: int = 25,
+        entity_type_prefixes: frozenset[str] | None = None,
+    ) -> None:
+        self._name = name
+        self._domain = domain
+        self._min_query_length = min_query_length
+        self._max_names = max_names
+        self._top_k = top_k
+        self._entity_type_prefixes = entity_type_prefixes
+
+        # Memoized name list and choices, keyed by entity_type_prefixes.
+        # Built lazily on first generate() / warm(); never invalidated.
+        self._names_cache: list[tuple[str, str, str, bool, str]] | None = None
+        self._choices_cache: list[str] | None = None
+
+        # Guards the one-time build (mirrors SymSpellSource pattern).
+        self._build_lock = threading.Lock()
+        self._built = False
+
+    # ------------------------------------------------------------------
+    # CandidateSource properties
+    # ------------------------------------------------------------------
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @property
+    def reason_code(self) -> ReasonCode:
+        return ReasonCode.FUZZY_MATCH
+
+    def supports(self, domain_pack_id: str) -> bool:
+        return domain_pack_id == self._domain
+
+    @property
+    def requires_existing_candidates(self) -> bool:
+        return False
+
+    # ------------------------------------------------------------------
+    # Cache build (thread-safe)
+    # ------------------------------------------------------------------
+
+    def _ensure_built(self, store: Any) -> None:
+        """Build name and choices caches if not already built.
+
+        Double-checked locking: cheap lock-free fast-path once built.
+        Any exception during build is swallowed so ``generate()`` can
+        return ``[]`` gracefully (never raise contract).
+        """
+        if self._built:
+            return
+        with self._build_lock:
+            if self._built:
+                return
+            try:
+                names = list(
+                    store.iter_suggest_names(
+                        entity_type_prefixes=self._entity_type_prefixes
+                    )
+                )
+                choices = [row[0] for row in names]
+                self._names_cache = names
+                self._choices_cache = choices
+            except Exception as exc:
+                logger.debug(
+                    "FuzzyRetrievalBruteSource '%s': iter_suggest_names failed: %s",
+                    self._name,
+                    exc,
+                )
+                # Leave _built=False so callers see None caches and return [].
+                return
+            self._built = True
+
+    def warm(self) -> None:
+        """No-op: cache is built on first ``generate()`` call.
+
+        ``CandidateSource.warm()`` receives no store reference, so the name
+        list cannot be pre-built here.  The build is deferred to the first
+        ``generate()`` call, which receives the store via ``ctx.store``.
+
+        Subclasses that have access to the store at warm time can override
+        ``warm()`` to call ``_ensure_built(store)`` directly.
+        """
+        # ``CandidateSource.warm()`` has no store parameter — actual build
+        # deferred to first generate() call.  The warm() contract still holds:
+        # if the runner calls warm() and then generate(), the first generate()
+        # call will build and subsequent calls skip the lock.
+
+    # ------------------------------------------------------------------
+    # generate
+    # ------------------------------------------------------------------
+
+    def generate(self, ctx: GenerationContext) -> list[CandidateEvidence]:
+        """Generate FUZZY-tier evidence via brute-force RapidFuzz.
+
+        Returns ``[]`` (never raises) on any internal failure.
+        """
+        try:
+            return self._generate_inner(ctx)
+        except Exception as exc:
+            logger.debug(
+                "FuzzyRetrievalBruteSource '%s': unexpected error for query '%s': %s",
+                self._name,
+                ctx.text_norm,
+                exc,
+            )
+            emit_candidates_generated(
+                ctx.trace,
+                self._name,
+                0,
+                entity_ids=[],
+                query=ctx.text_norm,
+                reason="error",
+            )
+            return []
+
+    def _generate_inner(self, ctx: GenerationContext) -> list[CandidateEvidence]:  # noqa: PLR0911
+        # 1. Cooperative deadline check.
+        if ctx.deadline is not None and time.monotonic() >= ctx.deadline:
+            return []
+
+        query_norm = ctx.text_norm
+
+        # 2. Min-length gate.
+        if len(query_norm) < self._min_query_length:
+            return []
+
+        # 3. Degenerate-token / punctuation-noise / single-letter check.
+        if _short_input_blocked(query_norm):
+            return []
+
+        # 4. Build / load caches.
+        self._ensure_built(ctx.store)
+
+        names = self._names_cache
+        choices = self._choices_cache
+        if names is None or choices is None:
+            # Build failed (iter_suggest_names raised or is not implemented).
+            emit_candidates_generated(
+                ctx.trace,
+                self._name,
+                0,
+                entity_ids=[],
+                query=query_norm,
+                reason="no_names",
+            )
+            return []
+
+        # 5. Cap check — skip brute-force on very large stores.
+        if len(names) > self._max_names:
+            emit_candidates_generated(
+                ctx.trace,
+                self._name,
+                0,
+                entity_ids=[],
+                query=query_norm,
+                reason="cap_exceeded",
+            )
+            return []
+
+        # 6. Brute-force fuzzy candidates.
+        raw = fuzzy_candidates(
+            query_norm,
+            names,
+            top_k=self._top_k,
+            choices=choices,
+        )
+
+        if not raw:
+            emit_candidates_generated(
+                ctx.trace,
+                self._name,
+                0,
+                entity_ids=[],
+                query=query_norm,
+            )
+            return []
+
+        # 7. Dedupe: keep best hit per entity_id (lowest typo_count wins, then
+        #    highest match_score, mirroring runner.suggest_prefix dedup logic).
+        best: dict[str, Any] = {}
+        for cand in raw:
+            eid = cand.entity_id
+            if eid not in best:
+                best[eid] = cand
+            else:
+                prev = best[eid]
+                if (cand.typo_count, -(cand.match_score or 0.0)) < (
+                    prev.typo_count,
+                    -(prev.match_score or 0.0),
+                ):
+                    best[eid] = cand
+
+        # 8. Map to CandidateEvidence, stamping match_tier=FUZZY explicitly.
+        evidence: list[CandidateEvidence] = []
+        for rank, cand in enumerate(best.values(), start=1):
+            evidence.append(
+                CandidateEvidence(
+                    entity_id=cand.entity_id,
+                    source_name=self._name,
+                    raw_score=(cand.match_score or 0.0) / 100.0,
+                    rank=rank,
+                    matched_field="fuzzy_retrieval",
+                    matched_value=cand.matched_value,
+                    match_tier=MatchTier.FUZZY,
+                )
+            )
+
+        emit_candidates_generated(
+            ctx.trace,
+            self._name,
+            len(evidence),
+            entity_ids=[e.entity_id for e in evidence],
+            query=query_norm,
+        )
+
+        return evidence

{resolvekit-0.1.3 → resolvekit-0.1.5}/src/resolvekit/shared/sources/symspell_base.py

@@ -34,6 +34,13 @@ SYMSPELL_BASE_SCORE = 0.9
 SYMSPELL_DISTANCE_PENALTY = 0.15
 SYMSPELL_MIN_SCORE = 0.5
 
+# Queries this short get a tighter edit-distance bound: two edits in a ≤6-char
+# string rewrite a third of it, which surfaces unrelated short names as
+# candidates ("paris" → "moris"/Mauritius at distance 2) rather than fixing
+# typos. Longer queries keep the source's configured max distance.
+SYMSPELL_SHORT_QUERY_MAX_LENGTH = 6
+SYMSPELL_SHORT_QUERY_MAX_DISTANCE = 1
+
 # Temp cache files older than this are presumed leaked by a crashed writer and
 # safe to reap; live builds finish in seconds.
 _STALE_TMP_AGE_SECONDS = 3600.0
@@ -645,6 +652,11 @@ class SymSpellSource(CandidateSource):
             edit_distance = suggestion.distance
             if corrected == text_norm:
                 continue
+            if (
+                len(text_norm) <= SYMSPELL_SHORT_QUERY_MAX_LENGTH
+                and edit_distance > SYMSPELL_SHORT_QUERY_MAX_DISTANCE
+            ):
+                continue
             entity_ids = store.lookup_name_exact(corrected, name_kinds=self._name_kinds)
             for entity_id in entity_ids:
                 rank += 1