bayesian_bm25_rs 0.1.1__tar.gz

bayesian_bm25_rs-0.1.1/.gitignore

@@ -0,0 +1,28 @@
+# Rust
+/target/
+
+# Python
+__pycache__/
+*.py[cod]
+*.pyd
+*.so
+*.dylib
+
+# Packaging
+/dist/
+/build/
+*.egg-info/
+
+# Virtualenv
+/.venv/
+/venv/
+
+# Cargo cache
+/.cargo/
+
+# Pyodide build env
+/.pyodide-xbuildenv-*/
+/.pyodide-venv/
+
+# OS
+.DS_Store

bayesian_bm25_rs-0.1.1/Cargo.lock

@@ -0,0 +1,133 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "bayesian_bm25"
+version = "0.1.1"
+dependencies = [
+ "pyo3",
+]
+
+[[package]]
+name = "heck"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
+
+[[package]]
+name = "libc"
+version = "0.2.180"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc"
+
+[[package]]
+name = "once_cell"
+version = "1.21.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
+
+[[package]]
+name = "portable-atomic"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.106"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "pyo3"
+version = "0.28.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fcf3ccafdf54c050be48a3a086d372f77ba6615f5057211607cd30e5ac5cec6d"
+dependencies = [
+ "libc",
+ "once_cell",
+ "portable-atomic",
+ "pyo3-build-config",
+ "pyo3-ffi",
+ "pyo3-macros",
+]
+
+[[package]]
+name = "pyo3-build-config"
+version = "0.28.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "972720a441c91fd9c49f212a1d2d74c6e3803b231ebc8d66c51efbd7ccab11c8"
+dependencies = [
+ "target-lexicon",
+]
+
+[[package]]
+name = "pyo3-ffi"
+version = "0.28.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5994456d9dab8934d600d3867571b6410f24fbd6002570ad56356733eb54859b"
+dependencies = [
+ "libc",
+ "pyo3-build-config",
+]
+
+[[package]]
+name = "pyo3-macros"
+version = "0.28.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "11ce9cc8d81b3c4969748807604d92b4eef363c5bb82b1a1bdb34ec6f1093a18"
+dependencies = [
+ "proc-macro2",
+ "pyo3-macros-backend",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "pyo3-macros-backend"
+version = "0.28.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eaf4b60036a154d23282679b658e3cc7d88d3b8c9a40b43824785f232d2e1b98"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "pyo3-build-config",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "syn"
+version = "2.0.114"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "target-lexicon"
+version = "0.13.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1dd07eb858a2067e2f3c7155d54e929265c264e6f37efe3ee7a8d1b5a1dd0ba"
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5"

bayesian_bm25_rs-0.1.1/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "bayesian_bm25"
+version = "0.1.1"
+edition = "2021"
+description = "Bayesian BM25 scoring and experimental validation (Rust core + Python bindings)"
+readme = "README.md"
+license = "UNLICENSED"
+repository = "https://github.com/sigridjineth/bayesian_bm25_rs"
+
+[lib]
+name = "bayesian_bm25"
+path = "src/lib.rs"
+crate-type = ["rlib", "cdylib"]
+
+[[bin]]
+name = "run_experiments"
+path = "src/main.rs"
+
+[dependencies]
+pyo3 = { version = "0.28", optional = true, features = ["extension-module"] }
+
+[features]
+python = ["pyo3"]

bayesian_bm25_rs-0.1.1/Makefile

@@ -0,0 +1,7 @@
+.PHONY: build clean
+
+build:
+	cargo build --release
+
+clean:
+	cargo clean

bayesian_bm25_rs-0.1.1/PKG-INFO

@@ -0,0 +1,112 @@
+Metadata-Version: 2.4
+Name: bayesian_bm25_rs
+Version: 0.1.1
+Classifier: License :: Other/Proprietary License
+Classifier: Programming Language :: Rust
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Operating System :: OS Independent
+Summary: Bayesian BM25 scoring and experimental validation (Rust core + Python bindings)
+Keywords: bm25,information-retrieval,search,ranking,bayesian,hybrid-search
+License: UNLICENSED
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Homepage, https://github.com/sigridjineth/bayesian_bm25_rs
+Project-URL: Issues, https://github.com/sigridjineth/bayesian_bm25_rs/issues
+Project-URL: Repository, https://github.com/sigridjineth/bayesian_bm25_rs
+
+# bb25 (Bayesian BM25)
+
+bb25 is a fast, self-contained BM25 + Bayesian calibration implementation with a minimal Python API. It also includes a small reference corpus and experiment suite so you can validate the expected numerical properties.
+
+- PyPI package name: `bayesian_bm25_rs`
+- Python import name: `bb25`
+
+## Install
+
+```
+pip install bayesian_bm25_rs
+```
+
+## Quick start
+
+### Use the built-in corpus and queries
+
+```
+import bb25 as bb
+
+corpus = bb.build_default_corpus()
+docs = corpus.documents()
+queries = bb.build_default_queries()
+
+bm25 = bb.BM25Scorer(corpus, 1.2, 0.75)
+score = bm25.score(queries[0].terms, docs[0])
+print("score0", score)
+```
+
+### Build your own corpus
+
+```
+import bb25 as bb
+
+corpus = bb.Corpus()
+corpus.add_document("d1", "neural networks for ranking", [0.1] * 8)
+corpus.add_document("d2", "bm25 is a strong baseline", [0.2] * 8)
+corpus.build_index()  # must be called before creating scorers
+
+bm25 = bb.BM25Scorer(corpus, 1.2, 0.75)
+print(bm25.idf("bm25"))
+```
+
+### Bayesian calibration + hybrid fusion
+
+```
+import bb25 as bb
+
+corpus = bb.build_default_corpus()
+docs = corpus.documents()
+queries = bb.build_default_queries()
+
+bm25 = bb.BM25Scorer(corpus, 1.2, 0.75)
+bayes = bb.BayesianBM25Scorer(bm25, 1.0, 0.5)
+vector = bb.VectorScorer()
+hybrid = bb.HybridScorer(bayes, vector)
+
+q = queries[0]
+prob_or = hybrid.score_or(q.terms, q.embedding, docs[0])
+prob_and = hybrid.score_and(q.terms, q.embedding, docs[0])
+print("OR", prob_or, "AND", prob_and)
+```
+
+## Run the experiments
+
+```
+import bb25 as bb
+
+results = bb.run_experiments()
+print(all(r.passed for r in results))
+```
+
+## Build from source (Rust)
+
+```
+make build
+```
+
+## PyPI publishing
+
+Build a wheel with maturin:
+
+```
+python -m pip install maturin
+maturin build --release
+```
+
+For Pyodide builds, see `docs/pyodide.md`.
+

bayesian_bm25_rs-0.1.1/README.md

@@ -0,0 +1,88 @@
+# bb25 (Bayesian BM25)
+
+bb25 is a fast, self-contained BM25 + Bayesian calibration implementation with a minimal Python API. It also includes a small reference corpus and experiment suite so you can validate the expected numerical properties.
+
+- PyPI package name: `bayesian_bm25_rs`
+- Python import name: `bb25`
+
+## Install
+
+```
+pip install bayesian_bm25_rs
+```
+
+## Quick start
+
+### Use the built-in corpus and queries
+
+```
+import bb25 as bb
+
+corpus = bb.build_default_corpus()
+docs = corpus.documents()
+queries = bb.build_default_queries()
+
+bm25 = bb.BM25Scorer(corpus, 1.2, 0.75)
+score = bm25.score(queries[0].terms, docs[0])
+print("score0", score)
+```
+
+### Build your own corpus
+
+```
+import bb25 as bb
+
+corpus = bb.Corpus()
+corpus.add_document("d1", "neural networks for ranking", [0.1] * 8)
+corpus.add_document("d2", "bm25 is a strong baseline", [0.2] * 8)
+corpus.build_index()  # must be called before creating scorers
+
+bm25 = bb.BM25Scorer(corpus, 1.2, 0.75)
+print(bm25.idf("bm25"))
+```
+
+### Bayesian calibration + hybrid fusion
+
+```
+import bb25 as bb
+
+corpus = bb.build_default_corpus()
+docs = corpus.documents()
+queries = bb.build_default_queries()
+
+bm25 = bb.BM25Scorer(corpus, 1.2, 0.75)
+bayes = bb.BayesianBM25Scorer(bm25, 1.0, 0.5)
+vector = bb.VectorScorer()
+hybrid = bb.HybridScorer(bayes, vector)
+
+q = queries[0]
+prob_or = hybrid.score_or(q.terms, q.embedding, docs[0])
+prob_and = hybrid.score_and(q.terms, q.embedding, docs[0])
+print("OR", prob_or, "AND", prob_and)
+```
+
+## Run the experiments
+
+```
+import bb25 as bb
+
+results = bb.run_experiments()
+print(all(r.passed for r in results))
+```
+
+## Build from source (Rust)
+
+```
+make build
+```
+
+## PyPI publishing
+
+Build a wheel with maturin:
+
+```
+python -m pip install maturin
+maturin build --release
+```
+
+For Pyodide builds, see `docs/pyodide.md`.

bayesian_bm25_rs-0.1.1/docs/pyodide.md

@@ -0,0 +1,26 @@
+# Pyodide Build Notes
+
+This project targets Pyodide by keeping the Python API pure (no file I/O, no OS dependencies) and using PyO3 for bindings. The actual build depends on your Pyodide toolchain setup.
+
+The Python import name is `bb25`.
+
+## Minimal Flow
+
+1) Install a Pyodide build toolchain (pyodide-build + Emscripten).
+2) From `bayesian_bm25_rs/`, run:
+
+```
+./scripts/build_pyodide.sh
+```
+
+This will place a wheel in `dist/`, which can be loaded in Pyodide.
+
+## Notes
+
+- If your Pyodide Python version is newer than the PyO3 version supports, set:
+
+```
+PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1
+```
+
+- For reproducibility, pin your Pyodide toolchain and Python version.

bayesian_bm25_rs-0.1.1/pyproject.toml

@@ -0,0 +1,35 @@
+[build-system]
+requires = ["maturin>=1.5,<2"]
+build-backend = "maturin"
+
+[project]
+name = "bayesian_bm25_rs"
+version = "0.1.1"
+description = "Bayesian BM25 scoring and experimental validation (Rust core + Python bindings)"
+readme = "README.md"
+requires-python = ">=3.8"
+license = { text = "UNLICENSED" }
+keywords = ["bm25", "information-retrieval", "search", "ranking", "bayesian", "hybrid-search"]
+classifiers = [
+  "License :: Other/Proprietary License",
+  "Programming Language :: Rust",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3 :: Only",
+  "Programming Language :: Python :: 3.8",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Operating System :: OS Independent",
+]
+
+[project.urls]
+Homepage = "https://github.com/sigridjineth/bayesian_bm25_rs"
+Repository = "https://github.com/sigridjineth/bayesian_bm25_rs"
+Issues = "https://github.com/sigridjineth/bayesian_bm25_rs/issues"
+
+[tool.maturin]
+bindings = "pyo3"
+module-name = "bb25"
+features = ["python"]

bayesian_bm25_rs-0.1.1/scripts/build_pyodide.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if ! command -v pyodide >/dev/null 2>&1; then
+  echo "pyodide CLI not found. Install pyodide-build and retry."
+  echo "Example: python -m pip install pyodide-build"
+  exit 1
+fi
+
+# Builds a wheel suitable for Pyodide.
+# You may need to set up a Pyodide toolchain and Emscripten beforehand.
+pyodide build --wheel -o dist

bayesian_bm25_rs-0.1.1/src/bayesian_scorer.rs

@@ -0,0 +1,79 @@
+use std::rc::Rc;
+
+use crate::bm25_scorer::BM25Scorer;
+use crate::corpus::Document;
+use crate::math_utils::{clamp, safe_log, safe_prob, sigmoid};
+
+pub struct BayesianBM25Scorer {
+    bm25: Rc<BM25Scorer>,
+    alpha: f64,
+    beta: f64,
+}
+
+impl BayesianBM25Scorer {
+    pub fn new(bm25: Rc<BM25Scorer>, alpha: f64, beta: f64) -> Self {
+        Self { bm25, alpha, beta }
+    }
+
+    pub fn likelihood(&self, score: f64) -> f64 {
+        sigmoid(self.alpha * (score - self.beta))
+    }
+
+    pub fn tf_prior(&self, tf: usize) -> f64 {
+        0.2 + 0.7 * (tf as f64 / 10.0).min(1.0)
+    }
+
+    pub fn norm_prior(&self, doc_length: usize, avg_doc_length: f64) -> f64 {
+        if avg_doc_length < 1.0 {
+            return 0.5;
+        }
+        let ratio = doc_length as f64 / avg_doc_length;
+        let prior = 1.0 / (1.0 + ratio);
+        clamp(prior, 0.1, 0.9)
+    }
+
+    pub fn composite_prior(&self, tf: usize, doc_length: usize, avg_doc_length: f64) -> f64 {
+        let p_tf = self.tf_prior(tf);
+        let p_norm = self.norm_prior(doc_length, avg_doc_length);
+        clamp(0.7 * p_tf + 0.3 * p_norm, 0.1, 0.9)
+    }
+
+    pub fn posterior(&self, score: f64, prior: f64) -> f64 {
+        let mut lik = self.likelihood(score);
+        lik = safe_prob(lik);
+        let prior = safe_prob(prior);
+        let numerator = lik * prior;
+        let denominator = numerator + (1.0 - lik) * (1.0 - prior);
+        numerator / denominator
+    }
+
+    pub fn score_term(&self, term: &str, doc: &Document) -> f64 {
+        let raw_score = self.bm25.score_term_standard(term, doc);
+        if raw_score == 0.0 {
+            return 0.0;
+        }
+        let tf = *doc.term_freq.get(term).unwrap_or(&0);
+        let prior = self.composite_prior(tf, doc.length, self.bm25.avgdl());
+        self.posterior(raw_score, prior)
+    }
+
+    pub fn score(&self, query_terms: &[String], doc: &Document) -> f64 {
+        let mut log_complement_sum = 0.0;
+        let mut has_match = false;
+
+        for term in query_terms {
+            let p = self.score_term(term, doc);
+            if p > 0.0 {
+                has_match = true;
+                let p = safe_prob(p);
+                log_complement_sum += safe_log(1.0 - p);
+            }
+        }
+
+        if !has_match {
+            return 0.0;
+        }
+
+        1.0 - log_complement_sum.exp()
+    }
+}

bayesian_bm25_rs-0.1.1/src/bm25_scorer.rs

@@ -0,0 +1,65 @@
+use std::rc::Rc;
+
+use crate::corpus::{Corpus, Document};
+
+pub struct BM25Scorer {
+    corpus: Rc<Corpus>,
+    k1: f64,
+    b: f64,
+}
+
+impl BM25Scorer {
+    pub fn new(corpus: Rc<Corpus>, k1: f64, b: f64) -> Self {
+        Self { corpus, k1, b }
+    }
+
+    pub fn idf(&self, term: &str) -> f64 {
+        let n = self.corpus.n as f64;
+        let df_t = *self.corpus.df.get(term).unwrap_or(&0) as f64;
+        ((n - df_t + 0.5) / (df_t + 0.5)).ln()
+    }
+
+    fn length_norm(&self, doc: &Document) -> f64 {
+        1.0 - self.b + self.b * (doc.length as f64) / self.corpus.avgdl
+    }
+
+    pub fn score_term_standard(&self, term: &str, doc: &Document) -> f64 {
+        let tf = *doc.term_freq.get(term).unwrap_or(&0) as f64;
+        if tf == 0.0 {
+            return 0.0;
+        }
+        let norm = self.length_norm(doc);
+        let idf_val = self.idf(term);
+        idf_val * (self.k1 + 1.0) * tf / (self.k1 * norm + tf)
+    }
+
+    pub fn score_term_rewritten(&self, term: &str, doc: &Document) -> f64 {
+        let tf = *doc.term_freq.get(term).unwrap_or(&0) as f64;
+        if tf == 0.0 {
+            return 0.0;
+        }
+        let norm = self.length_norm(doc);
+        let boost = (self.k1 + 1.0) * tf / (self.k1 * norm + tf);
+        let idf_val = self.idf(term);
+        idf_val * boost
+    }
+
+    pub fn score(&self, query_terms: &[String], doc: &Document) -> f64 {
+        query_terms
+            .iter()
+            .map(|term| self.score_term_standard(term, doc))
+            .sum()
+    }
+
+    pub fn upper_bound(&self, term: &str) -> f64 {
+        let idf_val = self.idf(term);
+        if idf_val <= 0.0 {
+            return 0.0;
+        }
+        (self.k1 + 1.0) * idf_val
+    }
+
+    pub fn avgdl(&self) -> f64 {
+        self.corpus.avgdl
+    }
+}

bayesian_bm25_rs-0.1.1/src/corpus.rs

@@ -0,0 +1,85 @@
+use std::collections::HashMap;
+
+use crate::tokenizer::Tokenizer;
+
+#[derive(Clone)]
+pub struct Document {
+    pub id: String,
+    pub text: String,
+    pub embedding: Vec<f64>,
+    pub tokens: Vec<String>,
+    pub length: usize,
+    pub term_freq: HashMap<String, usize>,
+}
+
+pub struct Corpus {
+    tokenizer: Tokenizer,
+    documents: Vec<Document>,
+    doc_by_id: HashMap<String, usize>,
+    pub n: usize,
+    pub avgdl: f64,
+    pub df: HashMap<String, usize>,
+}
+
+impl Corpus {
+    pub fn new(tokenizer: Tokenizer) -> Self {
+        Self {
+            tokenizer,
+            documents: Vec::new(),
+            doc_by_id: HashMap::new(),
+            n: 0,
+            avgdl: 0.0,
+            df: HashMap::new(),
+        }
+    }
+
+    pub fn add_document(&mut self, doc_id: &str, text: &str, embedding: Vec<f64>) {
+        let tokens = self.tokenizer.tokenize(text);
+        let mut term_freq = HashMap::new();
+        for token in &tokens {
+            *term_freq.entry(token.clone()).or_insert(0) += 1;
+        }
+
+        let doc = Document {
+            id: doc_id.to_string(),
+            text: text.to_string(),
+            embedding,
+            length: tokens.len(),
+            tokens,
+            term_freq,
+        };
+
+        let idx = self.documents.len();
+        self.documents.push(doc);
+        self.doc_by_id.insert(doc_id.to_string(), idx);
+    }
+
+    pub fn build_index(&mut self) {
+        self.n = self.documents.len();
+        self.df.clear();
+        let mut total_length = 0usize;
+
+        for doc in &self.documents {
+            total_length += doc.length;
+            for term in doc.term_freq.keys() {
+                *self.df.entry(term.clone()).or_insert(0) += 1;
+            }
+        }
+
+        self.avgdl = if self.n > 0 {
+            total_length as f64 / self.n as f64
+        } else {
+            0.0
+        };
+    }
+
+    pub fn get_document(&self, doc_id: &str) -> Option<&Document> {
+        self.doc_by_id
+            .get(doc_id)
+            .and_then(|idx| self.documents.get(*idx))
+    }
+
+    pub fn documents(&self) -> &[Document] {
+        &self.documents
+    }
+}