footnote-mcp 1.0.0__py3-none-any.whl

footnote_mcp/__init__.py

@@ -0,0 +1,13 @@
+"""footnote-mcp: an MCP server for source-grounded web research.
+
+The runtime lives as submodules of this package (``server``, ``core``,
+``search``, ...). ``cli`` is the console-script entry point; the import of the
+heavy ``server`` module is deferred into the call so ``import footnote_mcp`` stays
+cheap.
+"""
+
+
+def cli():
+    from .server import cli as _cli
+
+    _cli()

footnote_mcp/__main__.py

@@ -0,0 +1,6 @@
+"""Allow ``python -m footnote_mcp`` to launch the server."""
+
+from . import cli
+
+if __name__ == "__main__":
+    cli()

footnote_mcp/confidence.py

@@ -0,0 +1,65 @@
+from __future__ import annotations
+
+from datetime import datetime
+
+
+def calculate_confidence(
+    chunk,
+    avg_relevance,
+    query=None,
+    *,
+    has_factual_data,
+):
+    """
+    Calculate confidence score for a chunk based on relevance, freshness,
+    factual density, source position, and text substance.
+
+    The source-domain whitelist was intentionally removed to avoid
+    hardcoded trust assumptions in library code.
+    """
+    score = 0
+    now = datetime.now()
+
+    rel = chunk["relevance"]
+    if rel > 0.7:
+        score += 3
+    elif rel > 0.4:
+        score += 2
+    elif rel > 0.2:
+        score += 1
+
+    if avg_relevance > 0 and rel > avg_relevance * 1.3:
+        score += 1
+
+    if chunk.get("pub_date"):
+        pub_date = chunk["pub_date"]
+        age_hours = (now - pub_date).total_seconds() / 3600
+
+        if age_hours < 24:
+            score += 3
+        elif age_hours < 24 * 7:
+            score += 2
+        elif age_hours < 24 * 30:
+            score += 1
+    else:
+        score += 1
+
+    if has_factual_data(chunk["text"]):
+        score += 2
+
+    if chunk["source_idx"] < 3:
+        score += 1
+    elif chunk["source_idx"] < 6:
+        score += 0.5
+
+    word_count = len(chunk["text"].split())
+    if word_count > 100:
+        score += 1
+    elif word_count > 50:
+        score += 0.5
+
+    if score >= 7:
+        return "HIGH"
+    if score >= 4:
+        return "MEDIUM"
+    return "LOW"

footnote_mcp/core.py

@@ -0,0 +1,65 @@
+"""
+Internal runtime settings and shared optional dependency state.
+
+Implementation code lives in focused modules: search, fetch, extract,
+rerank, and pipeline.
+"""
+
+from __future__ import annotations
+
+# Optional dependencies
+try:
+    import trafilatura  # noqa: F401
+
+    HAS_TRAFILATURA = True
+except ImportError:
+    HAS_TRAFILATURA = False
+
+try:
+    from sentence_transformers import CrossEncoder, SentenceTransformer
+
+    HAS_EMBEDDINGS = True
+    HAS_CROSS_ENCODER = True
+except ImportError:
+    SentenceTransformer = None
+    CrossEncoder = None
+    HAS_EMBEDDINGS = False
+    HAS_CROSS_ENCODER = False
+
+
+# Configurable runtime knobs
+NUM_PER_ENGINE = 15
+TOP_N_FETCH = 15
+MAX_CONTENT_CHARS = 12000
+CHUNK_SIZE = 600
+CHUNK_OVERLAP = 100
+TOP_CHUNKS_PER_PAGE = 3
+TOTAL_CONTEXT_CHUNKS = 15
+FETCH_WORKERS = 10
+FETCH_TIMEOUT = 12
+
+IMPERSONATE = [
+    "chrome110",
+    "chrome116",
+    "chrome120",
+    "chrome123",
+    "chrome124",
+    "safari15_5",
+    "safari17_0",
+]
+
+__all__ = [
+    "HAS_TRAFILATURA",
+    "HAS_EMBEDDINGS",
+    "HAS_CROSS_ENCODER",
+    "NUM_PER_ENGINE",
+    "TOP_N_FETCH",
+    "MAX_CONTENT_CHARS",
+    "CHUNK_SIZE",
+    "CHUNK_OVERLAP",
+    "TOP_CHUNKS_PER_PAGE",
+    "TOTAL_CONTEXT_CHUNKS",
+    "FETCH_WORKERS",
+    "FETCH_TIMEOUT",
+    "IMPERSONATE",
+]

footnote_mcp/diagnostics.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+import logging
+import os
+import sys
+
+
+def get_logger() -> logging.Logger:
+    logger = logging.getLogger("footnote")
+    if not logger.handlers:
+        handler = logging.StreamHandler(sys.stderr)
+        handler.setFormatter(logging.Formatter("[%(name)s] %(levelname)s: %(message)s"))
+        logger.addHandler(handler)
+        logger.propagate = False
+
+    level_name = os.environ.get("FOOTNOTE_LOG_LEVEL", "WARNING").upper()
+    logger.setLevel(getattr(logging, level_name, logging.WARNING))
+    return logger
+
+
+log = get_logger()

footnote_mcp/extract.py

@@ -0,0 +1,216 @@
+from __future__ import annotations
+
+import json
+import re
+from collections import Counter
+
+from bs4 import BeautifulSoup
+
+from .diagnostics import log
+
+
+def _fallback_extract(html):
+    # ponytail: raw get_text(). trafilatura covers 95%+ of pages, this is the last resort.
+    # ceiling: naive get_text() chunking. upgrade: readability-lxml when fallback rate exceeds 10% of fetches.
+    return BeautifulSoup(html, "html.parser").get_text("\n", strip=True)
+
+
+def _extract_publish_date(html):
+    from datetime import datetime
+
+    soup = BeautifulSoup(html, "html.parser")
+    date_str = None
+
+    meta_selectors = [
+        ("meta", {"property": "article:published_time"}),
+        ("meta", {"name": "publication_date"}),
+        ("meta", {"name": "publishdate"}),
+        ("meta", {"property": "og:published_time"}),
+        ("meta", {"name": "date"}),
+        ("meta", {"itemprop": "datePublished"}),
+        ("time", {"datetime": True}),
+    ]
+
+    for tag, attrs in meta_selectors:
+        elem = soup.find(tag, attrs)
+        if elem:
+            date_str = elem.get("content") or elem.get("datetime")
+            if date_str:
+                break
+
+    if not date_str:
+        for script in soup.find_all("script", {"type": "application/ld+json"}):
+            try:
+                data = json.loads(script.string)
+                if isinstance(data, dict):
+                    date_str = data.get("datePublished") or data.get("dateCreated")
+                    if date_str:
+                        break
+            except Exception:
+                pass
+
+    if date_str:
+        try:
+            if "T" in date_str or "-" in date_str:
+                date_str = date_str.split("+")[0].split("Z")[0]
+                return datetime.fromisoformat(date_str.replace("T", " ")[:19])
+        except Exception:
+            pass
+
+    return None
+
+
+def extract_content(html, url=None):
+    from . import core
+
+    if core.HAS_TRAFILATURA:
+        text = core.trafilatura.extract(
+            html,
+            url=url,
+            include_comments=False,
+            include_tables=True,
+            no_fallback=False,
+            favor_recall=True,
+            output_format="markdown",
+        )
+        if text and len(text) > 100:
+            return text
+
+    return _fallback_extract(html)
+
+
+def chunk_text(text, chunk_size=None, overlap=None, lang="en"):
+    from . import core
+
+    if chunk_size is None:
+        chunk_size = core.CHUNK_SIZE
+    if overlap is None:
+        overlap = core.CHUNK_OVERLAP
+    if not text:
+        return []
+
+    paragraphs = re.split(r"\n{2,}", text.strip())
+    chunks = []
+    current = ""
+
+    for para in paragraphs:
+        para = para.strip()
+        if not para:
+            continue
+
+        if current and len(current) + len(para) + 2 > chunk_size:
+            chunks.append(current.strip())
+            if overlap > 0 and len(current) > overlap:
+                current = current[-overlap:] + "\n\n" + para
+            else:
+                current = para
+        else:
+            current = current + "\n\n" + para if current else para
+
+        while len(current) > chunk_size * 1.5:
+            split_pos = chunk_size
+            for delim in [". ", "! ", "? ", ".\n", ";\n", "\n"]:
+                pos = current.rfind(delim, 0, chunk_size + 50)
+                if pos > chunk_size * 0.3:
+                    split_pos = pos + len(delim)
+                    break
+
+            chunk_part = current[:split_pos].strip()
+            if chunk_part:
+                chunks.append(chunk_part)
+
+            remainder = current[split_pos:].strip()
+            if overlap > 0 and len(chunk_part) > overlap:
+                current = chunk_part[-overlap:] + " " + remainder
+            else:
+                current = remainder
+
+    if current.strip():
+        chunks.append(current.strip())
+
+    return [chunk for chunk in chunks if len(chunk) > 40]
+
+
+def _is_incomplete_chunk(text):
+    """Only reject clear mid-sentence fragments."""
+    if not text or len(text) < 30:
+        return True
+
+    text_stripped = text.strip()
+    # starts mid-sentence: lowercase letter
+    if len(text_stripped) < 100 and text_stripped[0].islower():
+        return True
+    # starts with punctuation (continuation)
+    if text_stripped[0] in ".,-—–…":
+        return True
+    # trailing dash = cut off mid-word
+    if text_stripped.endswith("-") or text_stripped.endswith("—"):
+        return True
+    return False
+
+
+
+def _is_low_quality_chunk(text):
+    """One-pass quality check. ponytail: merged _remove_garbage_lines patterns."""
+    if not text or len(text) < 30:
+        return True
+
+    words = text.split()
+    if not words:
+        return True
+    if len(words) / len(text) < 0.08:
+        return True
+
+    text_lower = text.lower()
+    # merged garbage + boilerplate patterns
+    garbage = [
+        "подписаться", "подпис", "subscribe", "sign up", "telegram", "whatsapp",
+        "вконтакте", "следите за нами", "follow us", "поделиться",
+        "share on", "tweet", "facebook", "twitter", "комментар", "comment",
+        "оставьте отзыв", "читайте также", "read also", "related articles",
+        "рекомендуем", "recommended", "похожие статьи", "subscribe now",
+        "email updates", "daily digest",
+        "cookie policy", "privacy policy", "terms of service", "all rights reserved", "© 20",
+        "copyright ©", "sign up for", "follow us on",
+        "share this article", "advertisement", "sponsored content", "click here to",
+        "read more »", "loading...", "please wait", "javascript is disabled",
+        "enable javascript", "accept cookies", "we use cookies",
+        "in your inbox", "sign up for our", "get the latest",
+    ]
+    if sum(1 for p in garbage if p in text_lower) >= 2:
+        return True
+
+    word_counts = Counter(w.lower() for w in words if len(w) > 3)
+    if word_counts and max(word_counts.values()) > len(words) * 0.3:
+        return True
+
+    long_words = [w for w in words if len(w) > 4]
+    if len(long_words) / len(words) < 0.2:
+        return True
+    return False
+
+
+def is_content_page(text, query=None, lang="en", min_avg_sentence_len=25, min_sentences=3):
+    """Return True when extracted text has enough sentence or line-level substance."""
+    if not text or len(text) < 100:
+        return False
+
+    sentences = [s.strip() for s in re.split(r"[.!?]\s+", text) if len(s.strip()) > 20]
+    lines = [l.strip() for l in text.split("\n") if len(l.strip()) > 25]
+    if len(sentences) < min_sentences and len(lines) < min_sentences:
+        return False
+    all_parts = sentences + lines
+    avg_len = sum(len(s) for s in all_parts) / len(all_parts)
+    return avg_len >= min_avg_sentence_len or len(all_parts) >= 8  # enough distinct items = real content
+
+
+def filter_low_quality_chunks(chunks):
+    filtered = []
+    for chunk in chunks:
+        if not _is_incomplete_chunk(chunk) and not _is_low_quality_chunk(chunk):
+            filtered.append(chunk)
+
+    removed = len(chunks) - len(filtered)
+    if removed:
+        log.info("[FILTER] Removed %s low-quality/incomplete chunks", removed)
+    return filtered

footnote_mcp/fetch.py

@@ -0,0 +1,114 @@
+from __future__ import annotations
+
+import random
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+from curl_cffi import requests as http
+
+from .diagnostics import log
+
+
+def _imp():
+    from . import core
+
+    return random.choice(core.IMPERSONATE)
+
+
+def _headers(lang="en"):
+    return {
+        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+        "Accept-Language": f"{lang},en-US;q=0.7,en;q=0.3",
+        "Accept-Encoding": "gzip, deflate, br",
+        "DNT": "1",
+        "Connection": "keep-alive",
+        "Upgrade-Insecure-Requests": "1",
+        "Sec-Fetch-Dest": "document",
+        "Sec-Fetch-Mode": "navigate",
+        "Sec-Fetch-Site": "none",
+        "Sec-Fetch-User": "?1",
+    }
+
+
+def _get(url, lang="en", cookies=None, max_retries=2, timeout=15, extra_headers=None, proxies=None):
+    """Single HTTP GET with TLS impersonation and retries.
+
+    ``extra_headers`` (optional) are merged on top of the default browser headers,
+    which lets callers attach auth tokens or custom headers for gated pages.
+    ``proxies`` (optional) routes the request through a proxy, e.g.
+    ``{"http": "http://host:port", "https": "http://host:port"}``.
+    """
+    headers = _headers(lang)
+    if extra_headers:
+        headers.update({str(k): str(v) for k, v in extra_headers.items()})
+    last_err = None
+    for attempt in range(max_retries + 1):
+        try:
+            return http.get(
+                url,
+                headers=headers,
+                cookies=cookies,
+                impersonate=_imp(),
+                timeout=timeout,
+                allow_redirects=True,
+                proxies=proxies,
+            )
+        except Exception as exc:
+            last_err = exc
+            if attempt < max_retries:
+                time.sleep(random.uniform(0.5, 1.5))
+    raise last_err
+
+
+def fetch_page(url, lang="en"):
+    """Fetch a single page → return (url, raw_html, publish_date, error)."""
+    from . import core
+    from .extract import _extract_publish_date
+
+    try:
+        resp = _get(url, lang, timeout=core.FETCH_TIMEOUT, max_retries=1)
+        if resp.status_code == 200:
+            pub_date = _extract_publish_date(resp.text)
+            return (url, resp.text, pub_date, None)
+        return (url, None, None, f"HTTP {resp.status_code}")
+    except Exception as exc:
+        return (url, None, None, str(exc))
+
+
+def fetch_pages_parallel(urls, query=None, lang="en"):
+    """Fetch multiple pages in parallel and return extracted text by URL."""
+    from . import core
+    from .extract import extract_content, is_content_page
+
+    results = {}
+
+    with ThreadPoolExecutor(max_workers=core.FETCH_WORKERS) as pool:
+        futures = {pool.submit(fetch_page, url, lang): url for url in urls}
+
+        for future in as_completed(futures):
+            url = futures[future]
+            try:
+                fetched_url, html, pub_date, err = future.result()
+                if err:
+                    log.info("[FETCH] failed %s... - %s", url[:60], err)
+                    continue
+
+                try:
+                    text = extract_content(html, url=fetched_url)
+                    if text and len(text) > 50:
+                        if not is_content_page(text, query=query, lang=lang):
+                            log.info("[FETCH] rejected %s... - low-quality content", url[:60])
+                            continue
+
+                        text = text[:core.MAX_CONTENT_CHARS]
+                        results[fetched_url] = {"text": text, "pub_date": pub_date}
+                        date_str = pub_date.strftime("%Y-%m-%d") if pub_date else "unknown"
+                        log.info("[FETCH] ok %s... - %s chars, date=%s", url[:60], len(text), date_str)
+                    else:
+                        log.info("[FETCH] rejected %s... - empty after extraction", url[:60])
+                except Exception as extract_err:
+                    log.warning("[FETCH] extraction failed %s... - %s", url[:60], extract_err)
+            except Exception as exc:
+                log.warning("[FETCH] failed %s... - %s", url[:60], exc)
+
+    return results

footnote_mcp/pipeline.py

@@ -0,0 +1,177 @@
+from __future__ import annotations
+
+from .diagnostics import log
+
+
+def search_extract_rerank(query, num_fetch=None, lang="en", debug=False):
+    """
+    Full pipeline:
+      1. Search Bing+DDG → merge
+      2. Take top N results
+      3. Parallel fetch + extraction
+      4. Chunk extracted text
+      5. Rerank chunks vs query
+      6. Return ranked context chunks with metadata
+    """
+    from . import core
+    from .extract import chunk_text, filter_low_quality_chunks
+    from .fetch import fetch_pages_parallel
+    from .rerank import filter_results_by_relevance, rerank_chunks
+    from .search import search
+
+    if num_fetch is None:
+        num_fetch = core.TOP_N_FETCH
+
+    log.info("QUERY: %s", query)
+
+    all_results = search(query, num=20, lang=lang, debug=debug)
+    if not all_results:
+        return [], [], set()
+
+    threshold = 0.25 if lang == "ru" else 0.30
+    log.info("[PIPELINE] Pre-filtering %s results by semantic relevance (threshold=%s)", len(all_results), threshold)
+    all_results = filter_results_by_relevance(query, all_results, threshold=threshold, lang=lang)
+
+    if not all_results:
+        log.info("[PIPELINE] No relevant results after pre-filtering")
+        return [], [], set()
+
+    top_results = all_results[:num_fetch]
+    log.info("[PIPELINE] Top %s results to fetch:", len(top_results))
+    for i, result in enumerate(top_results, 1):
+        engines = ", ".join(result["engines"])
+        log.info("  %s. [%s] (score=%s) %s", i, engines, result["score"], result["title"][:60])
+        log.info("     %s", result["url"][:80])
+
+    log.info("[PIPELINE] Fetching %s pages in parallel", len(top_results))
+    urls = [result["url"] for result in top_results]
+    fetched = fetch_pages_parallel(urls, query=query, lang=lang)
+
+    if not fetched:
+        log.info("[PIPELINE] No pages fetched successfully; using snippets only")
+        chunks_with_meta = []
+        snippet_urls = set()
+        for i, result in enumerate(top_results):
+            if result["snippet"]:
+                chunks_with_meta.append(
+                    {
+                        "text": result["snippet"],
+                        "source_idx": i,
+                        "source_url": result["url"],
+                        "source_title": result["title"],
+                        "chunk_idx": 0,
+                    }
+                )
+                snippet_urls.add(result["url"])
+        ranked = rerank_chunks(query, chunks_with_meta, top_k=12, lang=lang)
+        return ranked, top_results, snippet_urls
+
+    log.info("[PIPELINE] Chunking %s pages", len(fetched))
+    all_chunks = []
+
+    for i, result in enumerate(top_results):
+        page_data = fetched.get(result["url"])
+        if not page_data:
+            if result["snippet"]:
+                all_chunks.append(
+                    {
+                        "text": result["snippet"],
+                        "source_idx": i,
+                        "source_url": result["url"],
+                        "source_title": result["title"],
+                        "chunk_idx": 0,
+                        "pub_date": None,
+                    }
+                )
+            continue
+
+        text = page_data["text"]
+        pub_date = page_data.get("pub_date")
+        chunks = filter_low_quality_chunks(chunk_text(text, lang=lang))
+        log.info("  Source %s: %s chunks from %s", i + 1, len(chunks), result["title"][:50])
+
+        for ci, chunk in enumerate(chunks):
+            all_chunks.append(
+                {
+                    "text": chunk,
+                    "source_idx": i,
+                    "source_url": result["url"],
+                    "source_title": result["title"],
+                    "chunk_idx": ci,
+                    "pub_date": pub_date,
+                }
+            )
+
+    log.info("  Total chunks: %s", len(all_chunks))
+
+    ranked = rerank_chunks(query, all_chunks, top_k=12, lang=lang)
+    log.info("  Selected top %s chunks:", len(ranked))
+    for chunk in ranked:
+        if core.HAS_EMBEDDINGS:
+            log.info(
+                "    src=%s chunk=%s rel=%.3f (bm25=%.3f sem=%.3f) - %s...",
+                chunk["source_idx"] + 1,
+                chunk["chunk_idx"],
+                chunk["relevance"],
+                chunk["bm25"],
+                chunk["semantic"],
+                chunk["text"][:50],
+            )
+        else:
+            log.info(
+                "    src=%s chunk=%s rel=%.4f - %s...",
+                chunk["source_idx"] + 1,
+                chunk["chunk_idx"],
+                chunk["relevance"],
+                chunk["text"][:60],
+            )
+
+    fetched_urls = set(fetched.keys()) if fetched else set()
+    return ranked, top_results, fetched_urls
+
+
+def build_llm_context(ranked_chunks, search_results, fetched_urls=None, renumber_sources=True):
+    if not ranked_chunks:
+        return "No relevant content found.", {}, {}
+
+    by_source = {}
+    skipped_ghost_sources = set()
+    for chunk in ranked_chunks:
+        idx = chunk["source_idx"]
+        url = chunk["source_url"]
+        if fetched_urls is not None and url not in fetched_urls:
+            skipped_ghost_sources.add((idx, chunk["source_title"], url))
+            continue
+        if idx not in by_source:
+            by_source[idx] = {"title": chunk["source_title"], "url": url, "chunks": []}
+        by_source[idx]["chunks"].append(chunk)
+
+    if skipped_ghost_sources:
+        log.info("[CONTEXT] Filtered out %s sources with failed fetch:", len(skipped_ghost_sources))
+        for idx, title, url in sorted(skipped_ghost_sources):
+            log.info("  [%s] %s... - %s", idx + 1, title[:60], url[:60])
+
+    filtered_sources = {}
+    for idx, source_data in by_source.items():
+        if source_data["chunks"]:
+            filtered_sources[idx] = source_data
+
+    by_source = filtered_sources
+    if not by_source:
+        return "No relevant sources found.", {}, {}
+
+    if renumber_sources:
+        source_mapping = {old_idx: new_idx + 1 for new_idx, old_idx in enumerate(sorted(by_source.keys()))}
+    else:
+        source_mapping = {idx: idx + 1 for idx in by_source.keys()}
+
+    parts = []
+    for old_idx in sorted(by_source.keys()):
+        src = by_source[old_idx]
+        src_num = source_mapping[old_idx]
+        parts.append(f"[{src_num}] {src['title']}")
+        for chunk in src["chunks"]:
+            parts.append(chunk["text"].replace("**", ""))
+        parts.append("")
+
+    return "\n".join(parts), source_mapping, by_source