llm-join 0.2.0__py3-none-any.whl

llm_join/__init__.py

@@ -0,0 +1,4 @@
+from llm_join.join import fuzzy_join
+
+__all__ = ["fuzzy_join"]
+__version__ = "0.1.0"

llm_join/config.py

@@ -0,0 +1,52 @@
+from dataclasses import dataclass, field
+from typing import Callable, Optional
+
+import numpy as np
+
+
+@dataclass
+class ColumnConfig:
+    left_col: str
+    right_col: str
+    embed_fn: Callable[[list[str]], np.ndarray]
+    context: str = ""
+    column_context: dict[str, str] = field(default_factory=dict)
+    top_k: int = 5
+    threshold: float = 0.7
+    batch_size: int = 32
+    embed_threshold: Optional[float] = None
+    max_llm_calls: Optional[int] = None
+    max_retries: int = 3
+
+    def __post_init__(self):
+        if not self.left_col:
+            raise ValueError("left_col must not be empty")
+        if not self.right_col:
+            raise ValueError("right_col must not be empty")
+        if not self.context or not self.context.strip():
+            raise ValueError(
+                "context must not be empty — describe what the columns represent and what kind of match to make. "
+                "Example: \"pharmaceutical drug names — match generic INN names to US brand names\""
+            )
+        if not 0.0 < self.threshold <= 1.0:
+            raise ValueError(f"threshold must be in (0, 1], got {self.threshold}")
+        if self.top_k < 1:
+            raise ValueError(f"top_k must be >= 1, got {self.top_k}")
+        if self.batch_size < 1:
+            raise ValueError(f"batch_size must be >= 1, got {self.batch_size}")
+        if self.embed_threshold is not None and not 0.0 < self.embed_threshold <= 1.0:
+            raise ValueError(f"embed_threshold must be in (0, 1], got {self.embed_threshold}")
+        if self.max_llm_calls is not None and self.max_llm_calls < 1:
+            raise ValueError(f"max_llm_calls must be >= 1, got {self.max_llm_calls}")
+        if self.max_retries < 0:
+            raise ValueError(f"max_retries must be >= 0, got {self.max_retries}")
+
+    @property
+    def context_str(self) -> str:
+        parts = []
+        if self.context:
+            parts.append(self.context)
+        for col in (self.left_col, self.right_col):
+            if col in self.column_context:
+                parts.append(f"{col}: {self.column_context[col]}")
+        return ". ".join(parts)

llm_join/join.py

@@ -0,0 +1,129 @@
+import warnings
+from typing import Callable, Optional, Union
+import pandas as pd
+
+from llm_join.config import ColumnConfig
+from llm_join.retriever import EmbeddingRetriever
+from llm_join.scorer import LLMScorer
+from llm_join.merger import Merger, MatchResult
+
+
+def fuzzy_join(
+    df1: pd.DataFrame,
+    df2: pd.DataFrame,
+    *,
+    left_on: Union[str, list[str]],
+    right_on: Union[str, list[str]],
+    llm: Callable,
+    embed_fn: Callable,
+    context: str,
+    column_context: Optional[dict] = None,
+    top_k: int = 5,
+    threshold: float = 0.7,
+    how: str = "inner",
+    batch_size: int = 32,
+    embed_threshold: Optional[float] = None,
+    max_llm_calls: Optional[int] = None,
+    max_retries: int = 3,
+    return_reasoning: bool = False,
+) -> pd.DataFrame:
+    # Normalise column names to single string (multi-col join concatenates values)
+    left_col, right_col, df1, df2 = _normalise_cols(df1, df2, left_on, right_on)
+
+    cfg = ColumnConfig(
+        left_col=left_col,
+        right_col=right_col,
+        embed_fn=embed_fn,
+        context=context,
+        column_context=column_context or {},
+        top_k=top_k,
+        threshold=threshold,
+        batch_size=batch_size,
+        embed_threshold=embed_threshold,
+        max_llm_calls=max_llm_calls,
+        max_retries=max_retries,
+    )
+
+    retriever = EmbeddingRetriever(embed_fn=cfg.embed_fn)
+    scorer = LLMScorer(llm, max_retries=cfg.max_retries)
+    merger = Merger()
+
+    left_vals = df1[left_col].astype(str).tolist()
+    right_vals = df2[right_col].astype(str).tolist()
+
+    # Use retrieve_with_scores for embed_threshold path
+    candidates_per_row = retriever.retrieve_with_scores(left_vals, right_vals, top_k=cfg.top_k)
+
+    matches: list[MatchResult] = []
+    llm_call_count = 0
+
+    for left_val, candidates_with_scores in zip(left_vals, candidates_per_row):
+        if not candidates_with_scores:
+            continue
+
+        # embed_threshold short-circuit
+        if cfg.embed_threshold is not None:
+            best_candidate, best_score = candidates_with_scores[0]  # already sorted by score desc
+            if best_score >= cfg.embed_threshold:
+                matches.append(MatchResult(
+                    left_val=left_val,
+                    right_val=best_candidate,
+                    score=best_score,
+                    reasoning="skipped — embed score above threshold",
+                    embed_rank=0,
+                    match_method="embed_threshold",
+                ))
+                continue
+            if best_score < (1.0 - cfg.embed_threshold):
+                warnings.warn(
+                    f"Row '{left_val}' skipped: top embed score {best_score:.3f} "
+                    f"below non-match threshold {1.0 - cfg.embed_threshold:.3f}",
+                    UserWarning,
+                    stacklevel=2,
+                )
+                continue
+
+        # max_llm_calls cap
+        if cfg.max_llm_calls is not None and llm_call_count >= cfg.max_llm_calls:
+            warnings.warn(
+                f"max_llm_calls={cfg.max_llm_calls} reached. "
+                "Remaining rows skipped. Result is partial.",
+                UserWarning,
+                stacklevel=2,
+            )
+            break
+
+        candidates = [c for c, _ in candidates_with_scores]
+        results = scorer.score(left_val, candidates, cfg.context_str, threshold=cfg.threshold)
+        llm_call_count += 1
+        if results is None:
+            # LLM failed all retries — fall back to highest-scoring embed candidate
+            best_candidate, best_embed_score = candidates_with_scores[0]
+            matches.append(MatchResult(
+                left_val=left_val,
+                right_val=best_candidate,
+                score=best_embed_score,
+                reasoning="LLM failed — embed rank-0 fallback used",
+                embed_rank=0,
+                match_method="embed_fallback",
+            ))
+        elif results:
+            matches.extend(results)
+
+    return merger.merge(df1, df2, left_col, right_col, matches, how=how, return_reasoning=return_reasoning)
+
+
+def _normalise_cols(df1, df2, left_on, right_on):
+    if isinstance(left_on, list):
+        col = "__left_key__"
+        df1 = df1.copy()
+        df1[col] = df1[left_on].astype(str).agg(" ".join, axis=1)
+        left_on = col
+    if isinstance(right_on, list):
+        col = "__right_key__"
+        df2 = df2.copy()
+        df2[col] = df2[right_on].astype(str).agg(" ".join, axis=1)
+        right_on = col
+    return left_on, right_on, df1, df2
+
+

llm_join/merger.py

@@ -0,0 +1,64 @@
+from dataclasses import dataclass
+import pandas as pd
+
+
+@dataclass
+class MatchResult:
+    left_val: str
+    right_val: str
+    score: float
+    reasoning: str
+    embed_rank: int = 0
+    match_method: str = "llm"  # "llm" or "embed_threshold"
+
+
+class Merger:
+    def merge(
+        self,
+        df1: pd.DataFrame,
+        df2: pd.DataFrame,
+        left_col: str,
+        right_col: str,
+        matches: list[MatchResult],
+        how: str = "inner",
+        return_reasoning: bool = False,
+    ) -> pd.DataFrame:
+        if how not in ("inner", "left", "right", "outer"):
+            raise ValueError(f"how must be inner/left/right/outer, got '{how}'")
+        if right_col in df1.columns and right_col != left_col:
+            raise ValueError(
+                f"right_col '{right_col}' already exists in df1; rename before merging"
+            )
+
+        if not matches:
+            empty = pd.DataFrame(columns=list(df1.columns) + [
+                c for c in df2.columns if c != right_col or right_col == left_col
+            ])
+            if how == "inner":
+                return empty
+            if how == "left":
+                return pd.merge(df1, df2.iloc[:0], left_on=left_col, right_on=right_col, how="left")
+            if how == "right":
+                return pd.merge(df1.iloc[:0], df2, left_on=left_col, right_on=right_col, how="right")
+            # outer: return both frames with NaN fills
+            return pd.merge(df1, df2, left_on=left_col, right_on=right_col, how="outer")
+
+        match_df = pd.DataFrame({
+            left_col: [m.left_val for m in matches],
+            right_col: [m.right_val for m in matches],
+            "_llm_score": [m.score for m in matches],
+            "_llm_reasoning": [m.reasoning for m in matches],
+            "_embed_rank": [m.embed_rank for m in matches],
+            "_match_method": [m.match_method for m in matches],
+        })
+
+        df2_with_key = df2.merge(match_df, on=right_col, how="left")
+        result = df1.merge(df2_with_key, left_on=left_col, right_on=left_col, how=how)
+
+        if not return_reasoning:
+            result = result.drop(
+                columns=["_llm_score", "_llm_reasoning", "_embed_rank", "_match_method"],
+                errors="ignore",
+            )
+
+        return result.reset_index(drop=True)

llm_join/prompts.py

@@ -0,0 +1,17 @@
+_TEMPLATE = """\
+You are a data matching assistant.
+{context_line}For each pair below, score how likely LEFT matches RIGHT (0.0–1.0).
+Respond ONLY as a JSON array with no extra text:
+[{{"index": 0, "score": 0.95, "reasoning": "brief explanation"}}, ...]
+
+Pairs:
+{pairs}"""
+
+
+def build_prompt(left_val: str, candidates: list[str], context_str: str) -> str:
+    context_line = f"Context: {context_str}\n" if context_str.strip() else ""
+    pairs = "\n".join(
+        f'{i}. LEFT: "{left_val}" | RIGHT: "{c}"'
+        for i, c in enumerate(candidates)
+    )
+    return _TEMPLATE.format(context_line=context_line, pairs=pairs)

llm_join/retriever.py

@@ -0,0 +1,68 @@
+from typing import Callable
+import faiss
+import numpy as np
+
+
+class EmbeddingRetriever:
+    def __init__(self, embed_fn: Callable[[list[str]], np.ndarray]):
+        self._embed_fn = embed_fn
+
+    def _embed(self, texts: list[str]) -> np.ndarray:
+        arr = self._embed_fn(texts)
+        return np.array(arr, dtype="float32")
+
+    def retrieve(
+        self,
+        query_vals: list[str],
+        corpus_vals: list[str],
+        top_k: int = 5,
+    ) -> list[list[str]]:
+        if not corpus_vals:
+            return [[] for _ in query_vals]
+
+        top_k = min(top_k, len(corpus_vals))
+        corpus_vecs = self._embed(corpus_vals)
+        query_vecs = self._embed(query_vals)
+
+        # L2-normalize for cosine similarity via inner product
+        faiss.normalize_L2(corpus_vecs)
+        faiss.normalize_L2(query_vecs)
+
+        dim = corpus_vecs.shape[1]
+        index = faiss.IndexFlatIP(dim)
+        index.add(corpus_vecs)
+
+        _, indices = index.search(query_vecs, top_k)
+
+        return [
+            [corpus_vals[i] for i in row if i >= 0]
+            for row in indices
+        ]
+
+    def retrieve_with_scores(
+        self,
+        query_vals: list[str],
+        corpus_vals: list[str],
+        top_k: int = 5,
+    ) -> list[list[tuple[str, float]]]:
+        """Returns list of (candidate, cosine_score) per query, sorted by score desc."""
+        if not corpus_vals:
+            return [[] for _ in query_vals]
+
+        top_k = min(top_k, len(corpus_vals))
+        corpus_vecs = self._embed(corpus_vals)
+        query_vecs = self._embed(query_vals)
+
+        faiss.normalize_L2(corpus_vecs)
+        faiss.normalize_L2(query_vecs)
+
+        dim = corpus_vecs.shape[1]
+        index = faiss.IndexFlatIP(dim)
+        index.add(corpus_vecs)
+
+        scores, indices = index.search(query_vecs, top_k)
+
+        return [
+            [(corpus_vals[idx], float(score)) for idx, score in zip(row_indices, row_scores) if idx >= 0]
+            for row_indices, row_scores in zip(indices, scores)
+        ]

llm_join/scorer.py

@@ -0,0 +1,159 @@
+import asyncio
+import json
+import re
+import time
+import warnings
+from typing import Callable, Optional
+
+from llm_join.merger import MatchResult
+from llm_join.prompts import build_prompt
+
+
+class LLMScorer:
+    def __init__(self, llm: Callable, max_retries: int = 3):
+        self._llm = llm
+        self._is_async = asyncio.iscoroutinefunction(llm)
+        self._max_retries = max_retries
+
+    def score(
+        self,
+        left_val: str,
+        candidates: list[str],
+        context_str: str,
+        threshold: float = 0.7,
+    ) -> Optional[list[MatchResult]]:
+        if self._is_async:
+            raise TypeError(
+                "LLM is async; call score_async() instead, or pass a sync callable."
+            )
+        prompt = build_prompt(left_val, candidates, context_str)
+        last_exc: Optional[Exception] = None
+        for attempt in range(self._max_retries + 1):
+            try:
+                raw = self._llm(prompt)
+                return self._parse(left_val, candidates, raw, threshold)
+            except Exception as exc:
+                last_exc = exc
+                if attempt < self._max_retries:
+                    wait = 2 ** attempt  # 1s, 2s, 4s, ...
+                    warnings.warn(
+                        f"LLM call failed for '{left_val}' (attempt {attempt + 1}/{self._max_retries + 1}): "
+                        f"{exc!r}. Retrying in {wait}s.",
+                        UserWarning,
+                        stacklevel=2,
+                    )
+                    time.sleep(wait)
+        warnings.warn(
+            f"LLM call failed for '{left_val}' after {self._max_retries + 1} attempts: {last_exc!r}. "
+            "Falling back to top embed candidate.",
+            UserWarning,
+            stacklevel=2,
+        )
+        return None  # signals LLM failure — caller applies embed fallback
+
+    async def score_async(
+        self,
+        left_val: str,
+        candidates: list[str],
+        context_str: str,
+        threshold: float = 0.7,
+    ) -> Optional[list[MatchResult]]:
+        prompt = build_prompt(left_val, candidates, context_str)
+        last_exc: Optional[Exception] = None
+        for attempt in range(self._max_retries + 1):
+            try:
+                if self._is_async:
+                    raw = await self._llm(prompt)
+                else:
+                    raw = self._llm(prompt)
+                return self._parse(left_val, candidates, raw, threshold)
+            except Exception as exc:
+                last_exc = exc
+                if attempt < self._max_retries:
+                    wait = 2 ** attempt
+                    warnings.warn(
+                        f"LLM call failed for '{left_val}' (attempt {attempt + 1}/{self._max_retries + 1}): "
+                        f"{exc!r}. Retrying in {wait}s.",
+                        UserWarning,
+                        stacklevel=2,
+                    )
+                    await asyncio.sleep(wait)
+        warnings.warn(
+            f"LLM call failed for '{left_val}' after {self._max_retries + 1} attempts: {last_exc!r}. "
+            "Falling back to top embed candidate.",
+            UserWarning,
+            stacklevel=2,
+        )
+        return None  # signals LLM failure — caller applies embed fallback
+
+    def _parse(
+        self,
+        left_val: str,
+        candidates: list[str],
+        raw: str,
+        threshold: float,
+    ) -> list[MatchResult]:
+        try:
+            # strip markdown code fences if present
+            cleaned = (
+                raw.strip()
+                .removeprefix("```json")
+                .removeprefix("```")
+                .removesuffix("```")
+                .strip()
+            )
+            parsed = json.loads(cleaned)
+        except (json.JSONDecodeError, ValueError):
+            # retry: find JSON array anywhere in response
+            match = re.search(r'\[.*?\]', raw, re.DOTALL)
+            if match:
+                try:
+                    parsed = json.loads(match.group())
+                except json.JSONDecodeError:
+                    warnings.warn(f"LLM returned malformed JSON for '{left_val}': {raw!r}")
+                    return []
+            else:
+                warnings.warn(f"LLM returned malformed JSON for '{left_val}': {raw!r}")
+                return []
+
+        if not isinstance(parsed, list):
+            warnings.warn(f"LLM returned non-array JSON for '{left_val}': {raw!r}")
+            return []
+
+        # Collect all valid scored items above threshold, deduplicated by index
+        seen_indices: set[int] = set()
+        scored = []
+        for item in parsed:
+            idx = item.get("index", -1)
+            if not (0 <= idx < len(candidates)):
+                continue
+            if idx in seen_indices:
+                continue  # LLM returned duplicate index — skip
+            seen_indices.add(idx)
+            score = float(item.get("score", 0.0))
+            if score >= threshold:
+                scored.append((score, idx, item.get("reasoning", "")))
+
+        if not scored:
+            return []
+
+        # Find best score, return ALL candidates that tie at that score
+        best_score = max(s for s, _, _ in scored)
+        tied = [(idx, reasoning) for score, idx, reasoning in scored if score == best_score]
+
+        # Annotate reasoning when multiple candidates tie — visible in return_reasoning output
+        tie_note = (
+            f" [tied: {len(tied)} candidates scored {best_score} — all joined]"
+            if len(tied) > 1 else ""
+        )
+
+        return [
+            MatchResult(
+                left_val=left_val,
+                right_val=candidates[idx],
+                score=best_score,
+                reasoning=reasoning + tie_note,
+                embed_rank=idx,
+            )
+            for idx, reasoning in tied
+        ]

llm_join-0.2.0.dist-info/METADATA

@@ -0,0 +1,631 @@
+Metadata-Version: 2.4
+Name: llm-join
+Version: 0.2.0
+Summary: Fuzzy join DataFrames using LLM scoring and embedding retrieval
+License: MIT
+Keywords: entity-resolution,fuzzy-join,llm,nlp,pandas
+Requires-Python: >=3.9
+Requires-Dist: faiss-cpu>=1.7
+Requires-Dist: numpy>=1.23
+Requires-Dist: pandas>=1.5
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.21; extra == 'dev'
+Requires-Dist: pytest>=7; extra == 'dev'
+Provides-Extra: sentence-transformers
+Requires-Dist: sentence-transformers>=2.2; extra == 'sentence-transformers'
+Description-Content-Type: text/markdown
+
+# llm-join
+
+**The pandas join that understands what your data means.**
+
+`pd.merge` joins on exact values. `llm-join` joins on *meaning* — using embeddings to find candidates and an LLM you already have to decide if they match.
+
+---
+
+## Table of Contents
+
+- [The Problem](#the-problem)
+- [Install](#install)
+- [Quick Start](#quick-start)
+- [Why llm-join](#why-llm-join)
+- [Real-World Use Cases](#real-world-use-cases)
+- [How It Works](#how-it-works)
+- [Cost & Scale](#cost--scale)
+  - [The problem with naive LLM joins](#the-problem-with-naive-llm-joins)
+  - [Stage 1: Embeddings narrow the search](#stage-1-embeddings-narrow-the-search-cheap)
+  - [Stage 2: LLM scores only the hard cases](#stage-2-llm-scores-only-the-hard-cases-accurate)
+  - [Real cost example](#real-cost-example)
+  - [Further cost controls](#further-cost-controls)
+- [Usage](#usage)
+  - [Basic join](#basic-join)
+  - [With domain context](#with-domain-context)
+  - [See why matches were made](#see-why-matches-were-made)
+  - [Control cost](#control-cost)
+  - [Left join (audit unmatched rows)](#left-join-audit-unmatched-rows)
+  - [Multi-column join key](#multi-column-join-key)
+  - [Chaining multiple joins](#chaining-multiple-joins)
+- [Works with Any LLM](#works-with-any-llm)
+- [Works with Any Embedding Function](#works-with-any-embedding-function)
+- [vs. Alternatives](#vs-alternatives)
+- [Parameters](#parameters)
+- [License](#license)
+
+```python
+from llm_join import fuzzy_join
+
+result = fuzzy_join(df1, df2, left_on="vendor", right_on="supplier_name", llm=my_llm, embed_fn=my_embed)
+```
+
+---
+
+## The Problem
+
+You have two DataFrames. Same data, different text:
+
+| Your system | Their system |
+|-------------|--------------|
+| `Goldman Sachs & Co.` | `The Goldman Sachs Group Inc` |
+| `Sony WH-1000XM5` | `SONY-WH1000XM5-BLK` |
+| `MSFT Q4 license renewal` | `Microsoft Enterprise Agreement Q4-2024` |
+| `Python programming` | `Python (language)` |
+
+`pd.merge` returns nothing. Fuzzy string matching gets the wrong answer. You end up writing custom logic — or doing it by hand.
+
+**llm-join solves this in one line.**
+
+---
+
+## Install
+
+```bash
+# From GitHub (not yet on PyPI)
+pip install git+https://github.com/adityabalki/llm-join.git
+
+# Or clone and install locally
+git clone https://github.com/adityabalki/llm-join.git
+cd llm-join
+pip install -e .
+
+# Or copy the wheel to air-gapped machines
+pip install llm_join-0.1.0-py3-none-any.whl
+```
+
+---
+
+## Quick Start
+
+```python
+import pandas as pd
+import openai
+from llm_join import fuzzy_join
+
+# Your data
+df1 = pd.DataFrame({
+    "vendor": ["Goldman Sachs & Co.", "Amazon Web Services", "Microsoft Corp"],
+    "spend": [1_200_000, 890_000, 340_000]
+})
+
+df2 = pd.DataFrame({
+    "supplier_name": ["The Goldman Sachs Group Inc", "Amazon.com Inc.", "Microsoft Corporation"],
+    "category": ["Finance", "Cloud", "Software"]
+})
+
+# Wire up any LLM you already use
+client = openai.OpenAI()
+def llm(prompt):
+    return client.chat.completions.create(
+        model="gpt-4o-mini",
+        messages=[{"role": "user", "content": prompt}]
+    ).choices[0].message.content
+
+# Wire up your embedding function
+import numpy as np
+def my_embed(texts):
+    response = client.embeddings.create(model="text-embedding-3-small", input=texts)
+    return np.array([d.embedding for d in response.data], dtype="float32")
+
+# Join (inner by default — only rows that matched)
+result = fuzzy_join(
+    df1, df2,
+    left_on="vendor",
+    right_on="supplier_name",
+    llm=llm,
+    embed_fn=my_embed,
+    context="company names — match legal entity variants and abbreviations",
+    how="inner",   # "inner" | "left" | "right" | "outer"
+)
+
+print(result)
+```
+
+| vendor | spend | supplier_name | category |
+|---|---:|---|---|
+| Goldman Sachs & Co. | 1,200,000 | The Goldman Sachs Group Inc | Finance |
+| Amazon Web Services | 890,000 | Amazon.com Inc. | Cloud |
+| Microsoft Corp | 340,000 | Microsoft Corporation | Software |
+
+---
+
+## Why llm-join
+
+### vs. `pd.merge`
+Exact string match only. Fails on any variation in naming.
+
+### vs. fuzzy string matching (`fuzzywuzzy`, `rapidfuzz`)
+Character similarity, not semantic meaning. `"iPhone 14 Pro"` vs `"iPhone 14 Pro Max"` scores high — but they are different products. `"CABLE-USBC-200CM-BLK"` vs `"USB-C charging cable 2m black"` scores near zero — even though they are the same item.
+
+### vs. embedding similarity alone
+Fast and cheap, but no reasoning. Can't explain *why* two values match or catch false positives confidently.
+
+### llm-join
+Embeddings narrow down candidates (fast, cheap). LLM makes the final call with context (accurate). You get the best of both.
+
+---
+
+## Real-World Use Cases
+
+| Domain | Left table | Right table | Problem |
+|--------|-----------|-------------|---------|
+| **Supply chain** | Buyer catalog SKU | Supplier SKU | Match products across 50+ vendor catalogs |
+| **Finance** | Expense report payee | GL account / vendor master | Reconcile transactions automatically |
+| **Legal / M&A** | Contract party name | Corporate registry | Identify true legal entity |
+| **Compliance** | Customer name | OFAC sanctions list | Sanctions screening at scale |
+| **Retail / e-commerce** | Marketplace product listing | Master product catalog | Deduplicate listings across 50+ sellers |
+| **Logistics** | Shipment description | Harmonized tariff code | Auto-classify goods at customs |
+| **E-commerce** | Marketplace listing | Master product catalog | Deduplicate across platforms |
+| **Research** | Author name | Citation database | Disambiguate authors |
+| **Government** | Vendor name | Tax registry | Consolidate procurement spend |
+| **Real estate** | Raw address input | Property records DB | Standardize and match addresses |
+
+---
+
+## How It Works
+
+**Example:** A retailer's internal purchase orders use plain English product names. Their supplier sends a catalog with SKU codes. `pd.merge` matches nothing. llm-join bridges the gap.
+
+```python
+orders_df = pd.DataFrame({"product_name": [
+    "USB-C charging cable 2m black",
+    "ergonomic mesh office chair",
+    "27-inch 4K monitor",
+], "qty": [500, 30, 12]})
+
+catalog_df = pd.DataFrame({"sku": [
+    "CABLE-USBC-200CM-BLK",
+    "CABLE-USBA-200CM-BLK",
+    "CHAIR-MESH-ERG-ADJUSTABLE",
+    "CHAIR-TASK-FIXED-BLK",
+    "MON-27-4K-IPS-HDMI2",
+], "unit_price": [8.99, 6.49, 349.00, 189.00, 429.00]})
+
+result = fuzzy_join(
+    orders_df, catalog_df,
+    left_on="product_name", right_on="sku",
+    llm=my_llm, embed_fn=my_embed,
+    context="procurement — match buyer product descriptions to supplier SKU codes",
+    top_k=3, threshold=0.7,
+)
+```
+
+### Step 1 — Embed both columns
+
+Every value is converted to a vector. No API call — pure math, milliseconds.
+
+| Value | Meaning captured in vector |
+|---|---|
+| `"USB-C charging cable 2m black"` | cable / USB-C / length / color |
+| `"ergonomic mesh office chair"` | seating / ergonomic / mesh |
+| `"27-inch 4K monitor"` | display / size / resolution |
+| `"CABLE-USBC-200CM-BLK"` | cable / USB-C / 200cm / black |
+| `"CHAIR-MESH-ERG-ADJUSTABLE"` | seating / mesh / ergonomic |
+| `"MON-27-4K-IPS-HDMI2"` | display / 27in / 4K |
+
+### Step 2 — FAISS retrieves top-K candidates (no LLM)
+
+For each left row, faiss finds the `top_k` closest right vectors by cosine similarity. Everything else is eliminated — no LLM call needed.
+
+**Query: `"USB-C charging cable 2m black"` → top_k=3**
+
+| Rank | Candidate | Embed Score | Reaches LLM? |
+|---:|---|---:|---|
+| 0 | `CABLE-USBC-200CM-BLK` | 0.89 | ✓ yes |
+| 1 | `CABLE-USBA-200CM-BLK` | 0.71 | ✓ yes |
+| 2 | `CHAIR-TASK-FIXED-BLK` | 0.34 | ✓ yes (shares "BLK") |
+| — | `CHAIR-MESH-ERG-ADJUSTABLE` | 0.11 | ✗ eliminated |
+| — | `MON-27-4K-IPS-HDMI2` | 0.08 | ✗ eliminated |
+
+**Query: `"ergonomic mesh office chair"` → top_k=3**
+
+| Rank | Candidate | Embed Score | Reaches LLM? |
+|---:|---|---:|---|
+| 0 | `CHAIR-MESH-ERG-ADJUSTABLE` | 0.91 | ✓ yes |
+| 1 | `CHAIR-TASK-FIXED-BLK` | 0.74 | ✓ yes |
+| 2 | `CABLE-USBC-200CM-BLK` | 0.19 | ✓ yes |
+| — | `MON-27-4K-IPS-HDMI2` | 0.06 | ✗ eliminated |
+| — | `CABLE-USBA-200CM-BLK` | 0.14 | ✗ eliminated |
+
+**Query: `"27-inch 4K monitor"` → top_k=3**
+
+| Rank | Candidate | Embed Score | Reaches LLM? |
+|---:|---|---:|---|
+| 0 | `MON-27-4K-IPS-HDMI2` | 0.94 | ✓ yes |
+| 1 | `CABLE-USBC-200CM-BLK` | 0.22 | ✓ yes |
+| 2 | `CHAIR-TASK-FIXED-BLK` | 0.17 | ✓ yes |
+| — | `CHAIR-MESH-ERG-ADJUSTABLE` | 0.09 | ✗ eliminated |
+| — | `CABLE-USBA-200CM-BLK` | 0.12 | ✗ eliminated |
+
+**Result: 3 LLM calls instead of 3 × 5 = 15 pair-by-pair calls.**
+
+### Step 3 — One LLM call per left row scores all candidates
+
+All top-K candidates go into a single prompt. LLM returns a JSON array — one API call, all candidates scored.
+
+**Prompt sent for `"USB-C charging cable 2m black"`:**
+```
+Context: procurement — match buyer product descriptions to supplier SKU codes
+
+LEFT: "USB-C charging cable 2m black"
+
+Score each candidate (0.0–1.0):
+0. CABLE-USBC-200CM-BLK
+1. CABLE-USBA-200CM-BLK
+2. CHAIR-TASK-FIXED-BLK
+```
+
+**LLM response:**
+```json
+[
+  {"index": 0, "score": 0.97, "reasoning": "USBC = USB-C, 200CM = 2m, BLK = black. Exact match on all three specs."},
+  {"index": 1, "score": 0.38, "reasoning": "Correct length and color but USBA is USB-A, not USB-C. Wrong connector type."},
+  {"index": 2, "score": 0.04, "reasoning": "This is a chair SKU. No relation to a cable."}
+]
+```
+
+**Apply threshold=0.7:**
+
+| Candidate | LLM Score | Decision |
+|---|---:|---|
+| `CABLE-USBC-200CM-BLK` | 0.97 | ✓ **best match — joined** |
+| `CABLE-USBA-200CM-BLK` | 0.38 | ✗ below threshold |
+| `CHAIR-TASK-FIXED-BLK` | 0.04 | ✗ below threshold |
+
+### Step 4 — Merge matched rows
+
+```python
+print(result)
+```
+
+| product_name | qty | sku | unit_price |
+|---|---:|---|---:|
+| USB-C charging cable 2m black | 500 | CABLE-USBC-200CM-BLK | 8.99 |
+| ergonomic mesh office chair | 30 | CHAIR-MESH-ERG-ADJUSTABLE | 349.00 |
+| 27-inch 4K monitor | 12 | MON-27-4K-IPS-HDMI2 | 429.00 |
+
+The LLM correctly rejected `CABLE-USBA-200CM-BLK` (wrong connector) even though embedding scored it 0.71 — this is the case where LLM reasoning earns its cost.
+
+---
+
+## Cost & Scale
+
+### The problem with naive LLM joins
+
+If you sent every possible pair to the LLM:
+
+| Left rows | Right rows | Pairs to score | Cost (gpt-4o-mini ~$0.30/1M tokens) |
+|-----------|------------|----------------|--------------------------------------|
+| 1,000 | 10,000 | 10,000,000 | ~$150 |
+| 10,000 | 100,000 | 1,000,000,000 | ~$15,000 |
+| 100,000 | 1,000,000 | 100,000,000,000 | impossible |
+
+**llm-join solves this with a two-stage pipeline.**
+
+### Stage 1: Embeddings narrow the search (cheap)
+
+Convert every value to a vector. Use faiss to find the top-K most similar candidates per row. This is pure math — no API calls, runs in milliseconds.
+
+| | |
+|---|---|
+| Input pairs | 10,000 × 100,000 = **1,000,000,000** |
+| After embed + faiss (`top_k=5`) | **50,000** candidate pairs |
+| Eliminated for free | **99.995%** of all pairs |
+
+### Stage 2: LLM scores only the hard cases (accurate)
+
+Your LLM sees a small batch of plausible candidates per row — not the full cross product. It scores each candidate; the highest score above `threshold` wins. One best match per left row.
+
+Query: `"USB-C charging cable 2m black"`
+
+| Rank | Candidate | LLM Score | Decision |
+|---:|---|---:|---|
+| 0 | `CABLE-USBC-200CM-BLK` | 0.97 | ✓ **best match — joined** |
+| 1 | `CABLE-USBC-100CM-BLK` | 0.71 | scored, not selected (wrong length) |
+| 2 | `CABLE-USBA-200CM-BLK` | 0.38 | scored, not selected (wrong connector) |
+| 3 | `CHAIR-TASK-FIXED-BLK` | 0.04 | ✗ below threshold |
+| 4 | `MON-27-4K-IPS-HDMI2` | 0.01 | ✗ below threshold |
+
+### Real cost example
+
+| Setup | LLM calls | Estimated cost |
+|-------|-----------|----------------|
+| 10k × 100k, top_k=5 | 50,000 | ~$0.75 |
+| 10k × 100k, top_k=3 | 30,000 | ~$0.45 |
+| 10k × 100k, embed_threshold=0.95 | ~5,000 (obvious matches skip LLM) | ~$0.08 |
+
+### Further cost controls
+
+```python
+result = fuzzy_join(
+    df1, df2,
+    left_on="vendor", right_on="supplier",
+    llm=my_llm,
+    embed_fn=my_embed,
+    top_k=3,                # fewer candidates = fewer LLM tokens per row
+    embed_threshold=0.95,   # skip LLM entirely if embedding match score > 0.95
+    max_llm_calls=1000,     # hard cap — warns and returns partial result if hit
+)
+```
+
+| Parameter | Effect |
+|-----------|--------|
+| `top_k=3` (default 5) | 40% fewer LLM tokens |
+| `embed_threshold=0.95` | Skip LLM for obvious matches — typically saves 30–60% |
+| `max_llm_calls=N` | Budget guard — never exceeds N LLM calls |
+
+---
+
+## Usage
+
+### Basic join
+
+```python
+result = fuzzy_join(
+    orders_df, catalog_df,
+    left_on="product_name",
+    right_on="sku",
+    llm=my_llm,
+    embed_fn=my_embed,
+    context="procurement — match buyer product descriptions to supplier SKU codes",
+)
+```
+
+### With domain context
+
+```python
+result = fuzzy_join(
+    orders_df, catalog_df,
+    left_on="product_name",
+    right_on="sku",
+    llm=my_llm,
+    embed_fn=my_embed,
+    context="procurement — match buyer product descriptions to supplier SKU codes",
+    column_context={
+        "product_name": "plain English product description written by a buyer",
+        "sku": "supplier stock-keeping unit code, typically uppercase with hyphens",
+    },
+)
+```
+
+### See why matches were made
+
+```python
+result = fuzzy_join(
+    df1, df2,
+    left_on="vendor",
+    right_on="supplier_name",
+    llm=my_llm,
+    embed_fn=my_embed,
+    return_reasoning=True,
+)
+
+print(result[["vendor", "supplier_name", "_llm_score", "_llm_reasoning", "_embed_rank", "_match_method"]])
+```
+
+| vendor | supplier_name | _llm_score | _llm_reasoning | _embed_rank | _match_method |
+|---|---|---:|---|---:|---|
+| Goldman Sachs & Co. | The Goldman Sachs Group Inc | 0.97 | same firm, legal name variant | 0 | llm |
+
+### Control cost
+
+```python
+result = fuzzy_join(
+    df1, df2,
+    left_on="vendor",
+    right_on="supplier_name",
+    llm=my_llm,
+    embed_fn=my_embed,
+    embed_threshold=0.95,   # skip LLM if embedding match is obvious
+    max_llm_calls=500,      # hard cap — warns and returns partial result if hit
+    top_k=3,                # fewer candidates = fewer LLM tokens
+)
+```
+
+### Left join (audit unmatched rows)
+
+`how="left"` keeps all left rows — unmatched ones get NaN right columns. Useful to see what the LLM failed to match.
+
+```python
+result = fuzzy_join(df1, df2, left_on="a", right_on="b", llm=my_llm, embed_fn=my_embed, how="left")
+
+# Rows with no match
+unmatched = result[result["b"].isna()]
+```
+
+> **Note:** `how="outer"` is useful for reconciliation — unmatched left rows are left values with no match above threshold; unmatched right rows are right values that were never selected as a best match for any left row. `cross` join is not supported (it would be the naive O(n×m) approach that llm-join is designed to avoid).
+
+### Multi-column join key
+
+```python
+# orders_df has separate "product_name" and "category" columns
+# catalog_df has a single "sku_description" column like "CABLE / USB-C / 200CM / BLK"
+
+result = fuzzy_join(
+    orders_df, catalog_df,
+    left_on=["product_name", "category"],   # concatenated: "USB-C cable 2m · electronics"
+    right_on="sku_description",
+    llm=my_llm,
+    embed_fn=my_embed,
+    context="procurement — match buyer product + category to supplier SKU description",
+)
+```
+
+### Chaining multiple joins
+
+Each `fuzzy_join` returns a regular DataFrame — pipe them like `pd.merge`.
+
+```python
+# df1: transactions  (vendor + product columns)
+# df2: vendor master
+# df3: product catalog
+
+# Step 1 — match vendors
+step1 = fuzzy_join(
+    df1, df2,
+    left_on="vendor",
+    right_on="supplier_name",
+    llm=my_llm,
+    embed_fn=my_embed,
+    how="left",
+    return_reasoning=True,
+)
+step1 = step1.rename(columns={
+    "_llm_score":    "_vendor_score",
+    "_llm_reasoning": "_vendor_reasoning",
+    "_match_method": "_vendor_method",
+})
+
+# Step 2 — match products on result of step 1
+result = fuzzy_join(
+    step1, df3,
+    left_on="product",
+    right_on="catalog_item",
+    llm=my_llm,
+    embed_fn=my_embed,
+    how="left",
+    return_reasoning=True,
+)
+# result now has _vendor_score + _llm_score without column collision
+```
+
+---
+
+## Works with Any LLM
+
+Pass any callable that takes a prompt string and returns a string.
+
+```python
+# OpenAI
+import openai
+client = openai.OpenAI()
+llm = lambda p: client.chat.completions.create(
+    model="gpt-4o-mini",
+    messages=[{"role": "user", "content": p}]
+).choices[0].message.content
+
+# Anthropic
+import anthropic
+client = anthropic.Anthropic()
+llm = lambda p: client.messages.create(
+    model="claude-opus-4-7", max_tokens=512,
+    messages=[{"role": "user", "content": p}]
+).content[0].text
+
+# Google Gemini
+import google.generativeai as genai
+model = genai.GenerativeModel("gemini-2.0-flash")
+llm = lambda p: model.generate_content(p).text
+
+# Ollama (local, free)
+import ollama
+llm = lambda p: ollama.chat(
+    model="llama3.2", messages=[{"role": "user", "content": p}]
+)["message"]["content"]
+
+# Any custom endpoint
+import requests
+llm = lambda p: requests.post(
+    "https://your-llm-api.com/chat",
+    json={"prompt": p}
+).json()["response"]
+```
+
+## Works with Any Embedding Function
+
+Pass any callable `(list[str]) -> np.ndarray` (shape `[n, dim]`, dtype `float32`).
+
+```python
+import numpy as np
+
+# OpenAI embeddings
+import openai
+client = openai.OpenAI()
+def my_embed(texts):
+    response = client.embeddings.create(model="text-embedding-3-small", input=texts)
+    return np.array([d.embedding for d in response.data], dtype="float32")
+
+# Cohere
+import cohere
+co = cohere.Client("YOUR_KEY")
+def my_embed(texts):
+    response = co.embed(texts=texts, model="embed-english-v3.0", input_type="search_document")
+    return np.array(response.embeddings, dtype="float32")
+
+# sentence-transformers (local)
+from sentence_transformers import SentenceTransformer
+model = SentenceTransformer("all-MiniLM-L6-v2")
+def my_embed(texts):
+    return model.encode(texts, convert_to_numpy=True).astype("float32")
+
+# Any custom endpoint
+import requests
+def my_embed(texts):
+    response = requests.post(
+        "https://your-embed-api.com/embed",
+        json={"texts": texts}
+    ).json()
+    return np.array(response["embeddings"], dtype="float32")
+```
+
+---
+
+## vs. Alternatives
+
+| | llm-join | pd.merge | fuzzywuzzy | Jellyjoin | LinkTransformer |
+|---|:---:|:---:|:---:|:---:|:---:|
+| Semantic matching | ✓ | ✗ | ✗ | ✓ | ✓ |
+| LLM makes final decision | ✓ | ✗ | ✗ | ✗ | optional |
+| Reasoning per match | ✓ | ✗ | ✗ | ✗ | ✗ |
+| Domain context injection | ✓ | ✗ | ✗ | ✗ | ✗ |
+| Bring-your-own LLM callable | ✓ | n/a | n/a | ✗ | ✗ |
+| Bring-your-own embed callable | ✓ | n/a | n/a | ✗ | ✗ |
+| Enterprise-safe (no forced downloads) | ✓ | ✓ | ✓ | partial | ✗ |
+| Hard cost cap (`max_llm_calls`) | ✓ | n/a | n/a | ✗ | ✗ |
+| License | **MIT** | BSD | MIT | MIT | **GPL-3.0** |
+| Install size | ~50 MB | ~20 MB | ~30 MB | ~60 MB | **~3 GB** |
+
+---
+
+## Parameters
+
+| Parameter | Default | Description |
+|-----------|---------|-------------|
+| `left_on` | required | Column name(s) in df1 |
+| `right_on` | required | Column name(s) in df2 |
+| `llm` | required | Callable `(prompt: str) -> str` — your LLM function |
+| `embed_fn` | required | Callable `(list[str]) -> np.ndarray` — your embedding function |
+| `context` | required | Domain context injected into LLM prompt — describe what the columns represent and what kind of match to make |
+| `column_context` | `{}` | Per-column context dict `{"col": "description"}` |
+| `top_k` | `5` | Embedding candidates retrieved per row before LLM scoring |
+| `batch_size` | `32` | Reserved for future LLM batching (passed through to config) |
+| `threshold` | `0.7` | Minimum LLM score (0–1) to accept a match |
+| `how` | `"inner"` | Join type: `inner` (matched pairs only) / `left` (all left rows, NaN where unmatched) / `right` (all right rows, NaN where no left row matched them) / `outer` (full picture — both unmatched left and unmatched right rows). |
+| `embed_threshold` | `None` | Skip LLM when embedding score is decisive (saves cost) |
+| `max_llm_calls` | `None` | Hard cap on LLM calls — returns partial result with warning if hit |
+| `max_retries` | `3` | Retry failed LLM calls with exponential backoff (1s, 2s, 4s…). Set `0` to disable. |
+| `return_reasoning` | `False` | Append `_llm_score`, `_llm_reasoning`, `_embed_rank`, `_match_method` columns. `_match_method` is `"llm"` or `"embed_threshold"` — tells you which rows skipped the LLM. |
+
+---
+
+## License
+
+MIT

llm_join-0.2.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+llm_join/__init__.py,sha256=6xpu-ZnwfWvtH-uOX3PY4jgAA1I5D8WSp8xL__kVyp4,85
+llm_join/config.py,sha256=ne_2cY1HFGkUMTa_YSRqU2W41LytM9PQQWYR8c48j2Q,2130
+llm_join/join.py,sha256=KTMzgIpdp9hypq7ahnv2XZBs0nzwkJ46vrAPHcRPgb8,4537
+llm_join/merger.py,sha256=S92_aMtgxQohUzqoVeTaqQBuq3zJ29L4xbzClf-0kzg,2306
+llm_join/prompts.py,sha256=soh7WKIKExgN8cSMFxkVKsOLGuYKGyTlXaLrVRF0KJQ,625
+llm_join/retriever.py,sha256=mY-Rx1TWpHOqTA7dxgxIN_KzW-mhasTf9OXhx1x0Fu8,2012
+llm_join/scorer.py,sha256=3zwMFfFigo6XoSDq0IgFrypRcsNd9OdGc2hW3iY-BhQ,5821
+llm_join-0.2.0.dist-info/METADATA,sha256=lCji78ey-p8GhSvaWYHJLjwmV02Ms1C5f6kSOnDHNQg,21237
+llm_join-0.2.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+llm_join-0.2.0.dist-info/RECORD,,

llm_join-0.2.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any