misakanet-core 2.7.0__tar.gz

misakanet_core-2.7.0/PKG-INFO

@@ -0,0 +1,68 @@
+Metadata-Version: 2.4
+Name: misakanet-core
+Version: 2.7.0
+Summary: The zero-dependency core protocol engine for MisakaNet Swarm Knowledge Network.
+Author-email: Ikalus1988 <sheldonisspark@gmail.com>
+Project-URL: Homepage, https://github.com/Ikalus1988/MisakaNet
+Project-URL: Bug Tracker, https://github.com/Ikalus1988/MisakaNet/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# misakanet-core
+
+**Zero-dependency BM25 search engine with RRF fusion** — extracted from [MisakaNet](https://github.com/Ikalus1988/MisakaNet).
+
+- Pure Python, stdlib only
+- BM25 ranking with configurable k1/b
+- Metadata-weighted scoring
+- RRF (Reciprocal Rank Fusion) for multi-query fusion
+- CJK-aware tokenization
+- Works in air-gapped environments
+
+## Installation
+
+```bash
+pip install misakanet-core
+```
+
+## Usage
+
+```python
+from misakanet_core import BM25, ScoredDocument, tokenize, rrf
+
+# Prepare corpus
+docs = [
+    ScoredDocument("doc1", tokenize("the cat sat on the mat")),
+    ScoredDocument("doc2", tokenize("the dog sat on the log")),
+    ScoredDocument("doc3", tokenize("cats and dogs are friends")),
+]
+
+# Build index and search
+engine = BM25(docs)
+results = engine.search("cat dog", top_k=5)
+
+for result in results:
+    print(f"{result.doc_id}: {result.score:.4f}")
+
+# Multi-query fusion with RRF
+from misakanet_core import SearchResult, rrf
+query1 = engine.search("cat")
+query2 = engine.search("dog")
+fused = rrf([query1, query2], top_k=3)
+```
+
+## Why not use elasticsearch / tantivy / whoosh?
+
+| | misakanet-core | elasticsearch | tantivy | whoosh |
+|---|---|---|---|---|
+| Dependencies | **Zero** | JVM | Rust toolchain | Pure Python |
+| Install time | 0.5s | 5min+ | 2min+ | 2s |
+| Air-gapped | ✅ | ❌ | ❌ | ✅ |
+| CJK support | ✅ | ✅ | ⚠️ | ⚠️ |
+
+## License
+
+MIT

misakanet_core-2.7.0/README.md

@@ -0,0 +1,55 @@
+# misakanet-core
+
+**Zero-dependency BM25 search engine with RRF fusion** — extracted from [MisakaNet](https://github.com/Ikalus1988/MisakaNet).
+
+- Pure Python, stdlib only
+- BM25 ranking with configurable k1/b
+- Metadata-weighted scoring
+- RRF (Reciprocal Rank Fusion) for multi-query fusion
+- CJK-aware tokenization
+- Works in air-gapped environments
+
+## Installation
+
+```bash
+pip install misakanet-core
+```
+
+## Usage
+
+```python
+from misakanet_core import BM25, ScoredDocument, tokenize, rrf
+
+# Prepare corpus
+docs = [
+    ScoredDocument("doc1", tokenize("the cat sat on the mat")),
+    ScoredDocument("doc2", tokenize("the dog sat on the log")),
+    ScoredDocument("doc3", tokenize("cats and dogs are friends")),
+]
+
+# Build index and search
+engine = BM25(docs)
+results = engine.search("cat dog", top_k=5)
+
+for result in results:
+    print(f"{result.doc_id}: {result.score:.4f}")
+
+# Multi-query fusion with RRF
+from misakanet_core import SearchResult, rrf
+query1 = engine.search("cat")
+query2 = engine.search("dog")
+fused = rrf([query1, query2], top_k=3)
+```
+
+## Why not use elasticsearch / tantivy / whoosh?
+
+| | misakanet-core | elasticsearch | tantivy | whoosh |
+|---|---|---|---|---|
+| Dependencies | **Zero** | JVM | Rust toolchain | Pure Python |
+| Install time | 0.5s | 5min+ | 2min+ | 2s |
+| Air-gapped | ✅ | ❌ | ❌ | ✅ |
+| CJK support | ✅ | ✅ | ⚠️ | ⚠️ |
+
+## License
+
+MIT

misakanet_core-2.7.0/pyproject.toml

@@ -0,0 +1,23 @@
+[build-system]
+requires = ["setuptools>=61.0.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "misakanet-core"
+version = "2.7.0"
+description = "The zero-dependency core protocol engine for MisakaNet Swarm Knowledge Network."
+readme = "README.md"
+authors = [
+    { name = "Ikalus1988", email = "sheldonisspark@gmail.com" }
+]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+requires-python = ">=3.10"
+dependencies = []
+
+[project.urls]
+"Homepage" = "https://github.com/Ikalus1988/MisakaNet"
+"Bug Tracker" = "https://github.com/Ikalus1988/MisakaNet/issues"

misakanet_core-2.7.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

misakanet_core-2.7.0/src/misakanet_core/__init__.py

@@ -0,0 +1,202 @@
+"""misakanet-core: Zero-dependency BM25 search engine.
+Extracted from MisakaNet — see https://github.com/Ikalus1988/MisakaNet
+"""
+from __future__ import annotations
+
+import math
+import re
+import json
+from typing import NamedTuple, Sequence
+from collections import Counter
+from dataclasses import dataclass, field
+
+__all__ = [
+    "BM25", "RRF", "SearchResult",
+    "tokenize", "compute_idf", "compute_bm25",
+]
+
+K1 = 1.5
+B = 0.75
+
+
+# ── Types ──
+
+
+class SearchResult(NamedTuple):
+    doc_id: str
+    score: float
+    metadata: dict
+
+
+@dataclass
+class ScoredDocument:
+    """Internal representation of a scored document."""
+    doc_id: str
+    tokens: list[str]
+    metadata: dict = field(default_factory=dict)
+    score: float = 0.0
+
+
+# ── Tokenization ──
+
+
+def tokenize(text: str, min_len: int = 1) -> list[str]:
+    """Split text into lower-case tokens, filtering short words.
+
+    Handles mixed Latin/CJK text. CJK characters are split individually
+    to maximize BM25 recall across different queries.
+    """
+    text = text.lower()
+    # Split CJK characters each into their own token
+    text = re.sub(r"([\u4e00-\u9fff])", r" \1 ", text)
+    tokens = re.findall(r"[a-z0-9]+|[\u4e00-\u9fff]", text)
+    return [t for t in tokens if len(t) >= min_len]
+
+
+# ── IDF Computation ──
+
+
+def compute_idf(
+    docs: Sequence[ScoredDocument],
+    smoothing: bool = True,
+) -> dict[str, float]:
+    """Compute Inverse Document Frequency for each token across a corpus."""
+    n = len(docs)
+    df: Counter[str] = Counter()
+    for doc in docs:
+        unique_tokens = set(doc.tokens)
+        df.update(unique_tokens)
+
+    eps = 1.0 if smoothing else 0.0
+    return {
+        token: math.log((n - freq + 0.5) / (freq + 0.5) + eps)
+        for token, freq in df.items()
+    }
+
+
+# ── BM25 Scoring ──
+
+
+def compute_bm25(
+    query_tokens: list[str],
+    doc_tokens: list[str],
+    idf: dict[str, float],
+    avg_dl: float,
+    k1: float = K1,
+    b: float = B,
+) -> float:
+    """Compute BM25 score for a single document against a query."""
+    dl = len(doc_tokens)
+    tf = Counter(doc_tokens)
+    score = 0.0
+    for qt in set(query_tokens):
+        if qt not in idf:
+            continue
+        freq = tf.get(qt, 0)
+        if freq == 0:
+            continue
+        numerator = freq * (k1 + 1)
+        denominator = freq + k1 * (1 - b + b * dl / avg_dl)
+        score += idf[qt] * numerator / denominator
+    return score
+
+
+# ── BM25 Engine ──
+
+
+class BM25:
+    """BM25 retrieval engine with configurable scoring.
+
+    Usage::
+
+        docs = [
+            ScoredDocument("doc1", tokenize("the cat sat on the mat")),
+            ScoredDocument("doc2", tokenize("the dog sat on the log")),
+        ]
+        engine = BM25(docs)
+        results = engine.search("cat mat", top_k=5)
+    """
+
+    def __init__(
+        self,
+        corpus: Sequence[ScoredDocument],
+        k1: float = K1,
+        b: float = B,
+    ):
+        self.corpus = list(corpus)
+        self.k1 = k1
+        self.b = b
+        self.avg_dl = (
+            sum(len(d.tokens) for d in self.corpus) / max(len(self.corpus), 1)
+        )
+        self.idf = compute_idf(self.corpus)
+
+    def search(
+        self,
+        query: str,
+        top_k: int = 10,
+        metadata_weight: float = 0.0,
+    ) -> list[SearchResult]:
+        """Search the corpus and return ranked results.
+
+        Args:
+            query: Raw query string (will be tokenized).
+            top_k: Maximum number of results to return.
+            metadata_weight: Weight for metadata score (0 = BM25 only).
+
+        Returns:
+            List of SearchResult ordered by descending score.
+        """
+        query_tokens = tokenize(query)
+        if not query_tokens:
+            return []
+
+        scored: list[SearchResult] = []
+        for doc in self.corpus:
+            bm25_score = compute_bm25(
+                query_tokens, doc.tokens, self.idf, self.avg_dl,
+                k1=self.k1, b=self.b,
+            )
+            meta_score = (
+                sum(doc.metadata.values()) * metadata_weight
+                if doc.metadata else 0.0
+            )
+            total = bm25_score + meta_score
+            scored.append(SearchResult(doc.doc_id, total, doc.metadata))
+
+        scored.sort(key=lambda x: x.score, reverse=True)
+        return scored[:top_k]
+
+
+# ── RRF (Reciprocal Rank Fusion) ──
+
+
+def rrf(
+    result_lists: list[list[SearchResult]],
+    k: int = 60,
+    top_k: int = 10,
+) -> list[SearchResult]:
+    """Fuse multiple ranked lists using Reciprocal Rank Fusion.
+
+    Args:
+        result_lists: Multiple ranked result lists from different queries.
+        k: RRF constant (default 60, per the original paper).
+        top_k: Maximum number of fused results.
+
+    Returns:
+        Fused ranking ordered by descending RRF score.
+    """
+    scores: dict[str, float] = {}
+    metadata: dict[str, dict] = {}
+
+    for rank_list in result_lists:
+        for rank, result in enumerate(rank_list, start=1):
+            scores[result.doc_id] = scores.get(result.doc_id, 0.0) + 1.0 / (k + rank)
+            if result.doc_id not in metadata:
+                metadata[result.doc_id] = result.metadata
+
+    fused = sorted(scores.items(), key=lambda x: -x[1])
+    return [
+        SearchResult(doc_id, score, metadata.get(doc_id, {}))
+        for doc_id, score in fused[:top_k]
+    ]

misakanet_core-2.7.0/src/misakanet_core.egg-info/PKG-INFO

@@ -0,0 +1,68 @@
+Metadata-Version: 2.4
+Name: misakanet-core
+Version: 2.7.0
+Summary: The zero-dependency core protocol engine for MisakaNet Swarm Knowledge Network.
+Author-email: Ikalus1988 <sheldonisspark@gmail.com>
+Project-URL: Homepage, https://github.com/Ikalus1988/MisakaNet
+Project-URL: Bug Tracker, https://github.com/Ikalus1988/MisakaNet/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# misakanet-core
+
+**Zero-dependency BM25 search engine with RRF fusion** — extracted from [MisakaNet](https://github.com/Ikalus1988/MisakaNet).
+
+- Pure Python, stdlib only
+- BM25 ranking with configurable k1/b
+- Metadata-weighted scoring
+- RRF (Reciprocal Rank Fusion) for multi-query fusion
+- CJK-aware tokenization
+- Works in air-gapped environments
+
+## Installation
+
+```bash
+pip install misakanet-core
+```
+
+## Usage
+
+```python
+from misakanet_core import BM25, ScoredDocument, tokenize, rrf
+
+# Prepare corpus
+docs = [
+    ScoredDocument("doc1", tokenize("the cat sat on the mat")),
+    ScoredDocument("doc2", tokenize("the dog sat on the log")),
+    ScoredDocument("doc3", tokenize("cats and dogs are friends")),
+]
+
+# Build index and search
+engine = BM25(docs)
+results = engine.search("cat dog", top_k=5)
+
+for result in results:
+    print(f"{result.doc_id}: {result.score:.4f}")
+
+# Multi-query fusion with RRF
+from misakanet_core import SearchResult, rrf
+query1 = engine.search("cat")
+query2 = engine.search("dog")
+fused = rrf([query1, query2], top_k=3)
+```
+
+## Why not use elasticsearch / tantivy / whoosh?
+
+| | misakanet-core | elasticsearch | tantivy | whoosh |
+|---|---|---|---|---|
+| Dependencies | **Zero** | JVM | Rust toolchain | Pure Python |
+| Install time | 0.5s | 5min+ | 2min+ | 2s |
+| Air-gapped | ✅ | ❌ | ❌ | ✅ |
+| CJK support | ✅ | ✅ | ⚠️ | ⚠️ |
+
+## License
+
+MIT

misakanet_core-2.7.0/src/misakanet_core.egg-info/SOURCES.txt

@@ -0,0 +1,8 @@
+README.md
+pyproject.toml
+src/misakanet_core/__init__.py
+src/misakanet_core.egg-info/PKG-INFO
+src/misakanet_core.egg-info/SOURCES.txt
+src/misakanet_core.egg-info/dependency_links.txt
+src/misakanet_core.egg-info/top_level.txt
+tests/test_core.py

misakanet_core-2.7.0/src/misakanet_core.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

misakanet_core-2.7.0/src/misakanet_core.egg-info/top_level.txt

@@ -0,0 +1 @@
+misakanet_core

misakanet_core-2.7.0/tests/test_core.py

@@ -0,0 +1,126 @@
+"""Tests for misakanet-core."""
+from misakanet_core import (
+    BM25, ScoredDocument, SearchResult,
+    tokenize, compute_idf, compute_bm25, rrf,
+)
+
+
+def test_tokenize_basic():
+    assert tokenize("Hello World") == ["hello", "world"]
+    assert tokenize("BM25检索") == ["bm25", "检", "索"]
+
+
+def test_tokenize_cjk():
+    """CJK characters are kept as individual tokens for BM25 matching."""
+    result = tokenize("中文测试")
+    assert "中" in result
+    assert "文" in result
+    assert "测" in result
+    assert "试" in result
+    assert len(result) == 4
+
+
+def test_compute_idf():
+    docs = [
+        ScoredDocument("d1", tokenize("cat sat on mat")),
+        ScoredDocument("d2", tokenize("dog sat on log")),
+        ScoredDocument("d3", tokenize("cat sat on log")),
+    ]
+    idf = compute_idf(docs)
+    assert idf["cat"] > idf["on"]  # rarer = higher IDF
+    assert idf["on"] < idf["cat"]  # appears in all 3, lower IDF
+    assert "sat" in idf
+
+
+def test_bm25_simple():
+    docs = [
+        ScoredDocument("d1", tokenize("cat sat on the mat")),
+        ScoredDocument("d2", tokenize("the dog sat on the log")),
+    ]
+    engine = BM25(docs)
+    results = engine.search("cat", top_k=5)
+    assert len(results) == 2
+    assert results[0].doc_id == "d1"  # d1 has "cat"
+
+
+def test_bm25_top_k():
+    docs = [ScoredDocument(f"d{i}", tokenize(f"word{i} "*3)) for i in range(10)]
+    engine = BM25(docs)
+    results = engine.search("word3 word7", top_k=2)
+    assert len(results) == 2
+    assert results[0].doc_id == "d3"
+    assert results[1].doc_id == "d7"
+
+
+def test_bm25_empty_query():
+    docs = [ScoredDocument("d1", tokenize("hello world"))]
+    engine = BM25(docs)
+    results = engine.search("")
+    assert results == []
+
+
+def test_bm25_empty_corpus():
+    engine = BM25([])
+    results = engine.search("anything")
+    assert results == []
+
+
+def test_rrf_basic():
+    # Use rankings where one item is clearly best
+    r1 = [
+        SearchResult("a", 1.0, {}),
+        SearchResult("b", 0.5, {}),
+        SearchResult("c", 0.3, {}),
+    ]
+    r2 = [
+        SearchResult("a", 0.9, {}),
+        SearchResult("d", 0.4, {}),
+        SearchResult("e", 0.2, {}),
+    ]
+    fused = rrf([r1, r2], k=60, top_k=3)
+    assert len(fused) <= 3
+    assert fused[0].doc_id == "a"  # rank 1 in both lists
+
+
+def test_rrf_empty():
+    assert rrf([], top_k=5) == []
+    assert rrf([[SearchResult("a", 1.0, {})], []], top_k=5) == [SearchResult("a", 1.0 / 61, {})]
+
+
+def test_rrf_top_k_limits():
+    r1 = [SearchResult(chr(ord('a')+i), 1.0 - i*0.1, {}) for i in range(5)]
+    r2 = [SearchResult(chr(ord('z')-i), 1.0 - i*0.1, {}) for i in range(5)]
+    fused = rrf([r1, r2], top_k=3)
+    assert len(fused) <= 3
+
+
+def test_metadata_scoring():
+    docs = [
+        ScoredDocument("d1", tokenize("cat mat sat"), metadata={"priority": 1.0}),
+        ScoredDocument("d2", tokenize("dog cat"), metadata={"priority": 0.0}),
+    ]
+    engine = BM25(docs)
+    # Without metadata weight, both match "cat" equally
+    # With metadata weight, d1 gets a boost
+    results_no_meta = engine.search("cat", top_k=5, metadata_weight=0.0)
+    results_with_meta = engine.search("cat", top_k=5, metadata_weight=0.5)
+    assert results_with_meta[0].doc_id == "d1"  # metadata boost
+    # Without metadata, scores are from BM25 only
+    assert results_no_meta[0].score <= results_with_meta[0].score
+
+
+def test_compute_bm25_score():
+    idf = {"cat": 0.5, "dog": 0.3}
+    avg_dl = 3.0
+    score = compute_bm25(["cat"], ["cat", "dog", "bird"], idf, avg_dl)
+    assert score > 0
+
+
+def test_multiple_queries_rrf():
+    """RRF across different query aspects should handle overlap."""
+    cat_results = [SearchResult("d1", 0.9, {}), SearchResult("d3", 0.7, {})]
+    dog_results = [SearchResult("d2", 0.8, {}), SearchResult("d3", 0.6, {})]
+    fused = rrf([cat_results, dog_results], top_k=3)
+    assert len(fused) >= 2
+    d3_count = sum(1 for r in fused if r.doc_id == "d3")
+    assert d3_count == 1  # no duplicates