llm-join 0.2.2__tar.gz → 0.2.3__tar.gz

{llm_join-0.2.2 → llm_join-0.2.3}/.gitignore

@@ -11,3 +11,4 @@ dist/
 .claude/
 docs/
 tests/
+notebooks/

{llm_join-0.2.2 → llm_join-0.2.3}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.4
 Name: llm-join
-Version: 0.2.2
+Version: 0.2.3
 Summary: Fuzzy join DataFrames using LLM scoring and embedding retrieval
 License: MIT
 Keywords: entity-resolution,fuzzy-join,llm,nlp,pandas
 Requires-Python: >=3.9
-Requires-Dist: faiss-cpu>=1.7
-Requires-Dist: numpy<2,>=1.23
+Requires-Dist: faiss-cpu>=1.8
+Requires-Dist: numpy>=1.23
 Requires-Dist: pandas>=1.5
 Provides-Extra: sentence-transformers
 Requires-Dist: sentence-transformers>=2.2; extra == 'sentence-transformers'

{llm_join-0.2.2 → llm_join-0.2.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "llm-join"
-version = "0.2.2"
+version = "0.2.3"
 description = "Fuzzy join DataFrames using LLM scoring and embedding retrieval"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -12,8 +12,8 @@ license = { text = "MIT" }
 keywords = ["fuzzy-join", "llm", "entity-resolution", "pandas", "nlp"]
 dependencies = [
     "pandas>=1.5",
-    "numpy>=1.23,<2",
-    "faiss-cpu>=1.7",
+    "numpy>=1.23",
+    "faiss-cpu>=1.8",
 ]
 
 [project.optional-dependencies]

llm_join-0.2.2/notebooks/test_llm_join.ipynb

@@ -1,404 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# llm-join — Test Notebook\n",
-    "\n",
-    "No API key needed. Uses:\n",
-    "- **Mock LLM** — scores candidates using word overlap + character similarity heuristics\n",
-    "- **Mock embed_fn** — TF-IDF style character n-gram vectors (numpy only, no sklearn)\n",
-    "\n",
-    "Swap the mock functions for your real LLM/embed when ready."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## 1. Install"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Install from local clone (run from repo root) or GitHub\n",
-    "# pip install -e ..\n",
-    "# pip install git+https://github.com/adityabalki/llm-join.git\n",
-    "\n",
-    "import sys, os\n",
-    "sys.path.insert(0, os.path.abspath(\"..\"))  # works if notebook is in /notebooks/"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## 2. Mock LLM and embed_fn (no API key needed)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import json\n",
-    "import re\n",
-    "import numpy as np\n",
-    "\n",
-    "\n",
-    "# ── Mock embed_fn ────────────────────────────────────────────────────────────\n",
-    "# Character n-gram (trigram) bag-of-words vectors.\n",
-    "# Good enough for faiss to retrieve plausible candidates.\n",
-    "\n",
-    "def get_trigrams(text: str) -> set:\n",
-    "    t = text.lower()\n",
-    "    return {t[i:i+3] for i in range(len(t) - 2)}\n",
-    "\n",
-    "def mock_embed(texts: list[str]) -> np.ndarray:\n",
-    "    \"\"\"Returns (n, 300) float32 array using hashed trigram features.\"\"\"\n",
-    "    dim = 300\n",
-    "    vecs = np.zeros((len(texts), dim), dtype=\"float32\")\n",
-    "    for i, text in enumerate(texts):\n",
-    "        for tg in get_trigrams(text):\n",
-    "            idx = hash(tg) % dim\n",
-    "            vecs[i, idx] += 1.0\n",
-    "        norm = np.linalg.norm(vecs[i])\n",
-    "        if norm > 0:\n",
-    "            vecs[i] /= norm\n",
-    "    return vecs\n",
-    "\n",
-    "\n",
-    "# ── Mock LLM ─────────────────────────────────────────────────────────────────\n",
-    "# Parses the prompt, scores each LEFT|RIGHT pair using:\n",
-    "#   - word overlap (Jaccard on lowercase words)\n",
-    "#   - character trigram overlap\n",
-    "# Returns valid JSON the scorer can parse.\n",
-    "\n",
-    "def word_jaccard(a: str, b: str) -> float:\n",
-    "    wa = set(re.sub(r'[^a-z0-9 ]', '', a.lower()).split())\n",
-    "    wb = set(re.sub(r'[^a-z0-9 ]', '', b.lower()).split())\n",
-    "    if not wa or not wb:\n",
-    "        return 0.0\n",
-    "    return len(wa & wb) / len(wa | wb)\n",
-    "\n",
-    "def trigram_overlap(a: str, b: str) -> float:\n",
-    "    ta, tb = get_trigrams(a), get_trigrams(b)\n",
-    "    if not ta or not tb:\n",
-    "        return 0.0\n",
-    "    return len(ta & tb) / len(ta | tb)\n",
-    "\n",
-    "def mock_llm(prompt: str) -> str:\n",
-    "    \"\"\"Parses llm-join prompt format and returns JSON scores.\"\"\"\n",
-    "    left_val = \"\"\n",
-    "    candidates = []\n",
-    "\n",
-    "    for line in prompt.splitlines():\n",
-    "        m = re.match(r'(\\d+)\\. LEFT: \"(.+?)\" \\| RIGHT: \"(.+?)\"', line)\n",
-    "        if m:\n",
-    "            if not left_val:\n",
-    "                left_val = m.group(2)\n",
-    "            candidates.append((int(m.group(1)), m.group(2), m.group(3)))\n",
-    "\n",
-    "    results = []\n",
-    "    for idx, left, right in candidates:\n",
-    "        wj = word_jaccard(left, right)\n",
-    "        tg = trigram_overlap(left, right)\n",
-    "        score = round(0.5 * wj + 0.5 * tg, 3)\n",
-    "        if score > 0.6:\n",
-    "            reasoning = \"strong word and character overlap\"\n",
-    "        elif score > 0.3:\n",
-    "            reasoning = \"partial match\"\n",
-    "        else:\n",
-    "            reasoning = \"low similarity\"\n",
-    "        results.append({\"index\": idx, \"score\": score, \"reasoning\": reasoning})\n",
-    "\n",
-    "    return json.dumps(results)\n",
-    "\n",
-    "\n",
-    "# Quick sanity check\n",
-    "print(mock_embed([\"Goldman Sachs\", \"Amazon\"]).shape)   # (2, 300)\n",
-    "print(mock_llm('0. LEFT: \"aspirin\" | RIGHT: \"Bayer Aspirin\"'))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## 3. Test Data"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import pandas as pd\n",
-    "\n",
-    "# Left: your internal vendor list\n",
-    "df1 = pd.DataFrame({\n",
-    "    \"vendor\":  [\"Goldman Sachs & Co.\", \"Amazon Web Services\", \"Microsoft Corp\",\n",
-    "                \"Apple Inc\", \"JP Morgan\", \"Unmatched Vendor Ltd\"],\n",
-    "    \"spend\":   [1_200_000, 890_000, 340_000, 500_000, 220_000, 99_000],\n",
-    "})\n",
-    "\n",
-    "# Right: supplier master (different naming conventions)\n",
-    "df2 = pd.DataFrame({\n",
-    "    \"supplier_name\": [\"The Goldman Sachs Group Inc\", \"Amazon.com Inc.\",\n",
-    "                      \"Microsoft Corporation\", \"Apple Incorporated\",\n",
-    "                      \"JPMorgan Chase & Co.\", \"Extra Supplier Not In Left\"],\n",
-    "    \"category\":      [\"Finance\", \"Cloud\", \"Software\", \"Hardware\", \"Finance\", \"Other\"],\n",
-    "})\n",
-    "\n",
-    "print(\"df1:\"); display(df1)\n",
-    "print(\"df2:\"); display(df2)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## 4. Basic Inner Join"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from llm_join import fuzzy_join\n",
-    "\n",
-    "result = fuzzy_join(\n",
-    "    df1, df2,\n",
-    "    left_on=\"vendor\",\n",
-    "    right_on=\"supplier_name\",\n",
-    "    llm=mock_llm,\n",
-    "    embed_fn=mock_embed,\n",
-    "    context=\"company vendor names — match legal entity variants and abbreviations\",\n",
-    "    how=\"inner\",\n",
-    "    threshold=0.3,   # lower threshold for mock (real LLM scores higher)\n",
-    ")\n",
-    "\n",
-    "display(result)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## 5. Left Join — See What Didn't Match"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "result_left = fuzzy_join(\n",
-    "    df1, df2,\n",
-    "    left_on=\"vendor\",\n",
-    "    right_on=\"supplier_name\",\n",
-    "    llm=mock_llm,\n",
-    "    embed_fn=mock_embed,\n",
-    "    how=\"left\",\n",
-    "    threshold=0.3,\n",
-    ")\n",
-    "\n",
-    "display(result_left)\n",
-    "\n",
-    "unmatched = result_left[result_left[\"supplier_name\"].isna()]\n",
-    "print(f\"\\nUnmatched left rows ({len(unmatched)}):\")\n",
-    "display(unmatched)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## 6. Return Reasoning — See Why Each Match Was Made"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "result_reasoned = fuzzy_join(\n",
-    "    df1, df2,\n",
-    "    left_on=\"vendor\",\n",
-    "    right_on=\"supplier_name\",\n",
-    "    llm=mock_llm,\n",
-    "    embed_fn=mock_embed,\n",
-    "    how=\"left\",\n",
-    "    threshold=0.3,\n",
-    "    return_reasoning=True,\n",
-    ")\n",
-    "\n",
-    "display(result_reasoned[[\"vendor\", \"supplier_name\", \"_llm_score\", \"_llm_reasoning\", \"_embed_rank\", \"_match_method\"]])"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## 7. Outer Join — Full Reconciliation (both gaps)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "result_outer = fuzzy_join(\n",
-    "    df1, df2,\n",
-    "    left_on=\"vendor\",\n",
-    "    right_on=\"supplier_name\",\n",
-    "    llm=mock_llm,\n",
-    "    embed_fn=mock_embed,\n",
-    "    how=\"outer\",\n",
-    "    threshold=0.3,\n",
-    ")\n",
-    "\n",
-    "display(result_outer)\n",
-    "\n",
-    "# Rows in supplier master with no match in your vendor list\n",
-    "unmatched_right = result_outer[result_outer[\"vendor\"].isna()]\n",
-    "print(f\"\\nRight rows with no left match ({len(unmatched_right)}):\")\n",
-    "display(unmatched_right)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## 8. Cost Controls"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import warnings\n",
-    "\n",
-    "result_capped = fuzzy_join(\n",
-    "    df1, df2,\n",
-    "    left_on=\"vendor\",\n",
-    "    right_on=\"supplier_name\",\n",
-    "    llm=mock_llm,\n",
-    "    embed_fn=mock_embed,\n",
-    "    top_k=3,              # fewer candidates per row\n",
-    "    threshold=0.3,\n",
-    "    max_llm_calls=3,      # cap at 3 LLM calls (will warn and return partial)\n",
-    "    how=\"left\",\n",
-    ")\n",
-    "\n",
-    "display(result_capped)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## 9. Chained Joins"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# df1: transactions with vendor + product\n",
-    "transactions = pd.DataFrame({\n",
-    "    \"vendor\":  [\"Goldman Sachs & Co.\", \"Microsoft Corp\"],\n",
-    "    \"product\": [\"equity trading platform\", \"office suite software\"],\n",
-    "    \"amount\":  [50_000, 12_000],\n",
-    "})\n",
-    "\n",
-    "product_catalog = pd.DataFrame({\n",
-    "    \"catalog_item\": [\"Equity Trading System\", \"Microsoft Office 365\", \"Azure Cloud\"],\n",
-    "    \"sku\":          [\"EQ-001\", \"MS-OFF-365\", \"AZ-001\"],\n",
-    "})\n",
-    "\n",
-    "# Step 1: match vendors\n",
-    "step1 = fuzzy_join(\n",
-    "    transactions, df2,\n",
-    "    left_on=\"vendor\", right_on=\"supplier_name\",\n",
-    "    llm=mock_llm, embed_fn=mock_embed,\n",
-    "    how=\"left\", threshold=0.3,\n",
-    "    return_reasoning=True,\n",
-    ")\n",
-    "step1 = step1.rename(columns={\n",
-    "    \"_llm_score\": \"_vendor_score\",\n",
-    "    \"_llm_reasoning\": \"_vendor_reasoning\",\n",
-    "    \"_match_method\": \"_vendor_method\",\n",
-    "    \"_embed_rank\": \"_vendor_embed_rank\",\n",
-    "})\n",
-    "\n",
-    "# Step 2: match products\n",
-    "result_chained = fuzzy_join(\n",
-    "    step1, product_catalog,\n",
-    "    left_on=\"product\", right_on=\"catalog_item\",\n",
-    "    llm=mock_llm, embed_fn=mock_embed,\n",
-    "    how=\"left\", threshold=0.1,\n",
-    "    return_reasoning=True,\n",
-    ")\n",
-    "\n",
-    "display(result_chained)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## 10. Swap to Real LLM / Embed\n",
-    "\n",
-    "When you have API access, replace the mock functions:\n",
-    "\n",
-    "```python\n",
-    "import openai, numpy as np\n",
-    "client = openai.OpenAI(api_key=\"...\")  # or base_url=\"https://your-enterprise-endpoint\"\n",
-    "\n",
-    "def my_llm(prompt):\n",
-    "    return client.chat.completions.create(\n",
-    "        model=\"gpt-4o-mini\",\n",
-    "        messages=[{\"role\": \"user\", \"content\": prompt}]\n",
-    "    ).choices[0].message.content\n",
-    "\n",
-    "def my_embed(texts):\n",
-    "    response = client.embeddings.create(model=\"text-embedding-3-small\", input=texts)\n",
-    "    return np.array([d.embedding for d in response.data], dtype=\"float32\")\n",
-    "\n",
-    "# Then use threshold=0.7 (default) instead of 0.3\n",
-    "result = fuzzy_join(df1, df2, left_on=\"vendor\", right_on=\"supplier_name\",\n",
-    "                    llm=my_llm, embed_fn=my_embed)\n",
-    "```"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "name": "python",
-   "version": "3.12.0"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 4
-}