conflens 0.1.14__py3-none-any.whl

conflens/bibtex.py

@@ -0,0 +1,56 @@
+"""Build a BibTeX bibliography from analysis results."""
+
+from __future__ import annotations
+
+import re
+
+from .models import AnalysisResult, Paper
+
+_YEAR = re.compile(r"(19|20)\d{2}")
+
+
+def _cite_key(paper: Paper) -> str:
+    key = re.sub(r"[^0-9A-Za-z]", "", paper.paper_id)
+    if not key:
+        key = "paper"
+    if not key[0].isalpha():
+        key = "p" + key
+    return key
+
+
+def _year(paper: Paper) -> str:
+    m = _YEAR.search(paper.paper_id)
+    return m.group(0) if m else ""
+
+
+def _clean(value: str) -> str:
+    # Strip characters that would break a BibTeX field; keep it simple and safe.
+    return value.replace("{", "").replace("}", "").replace("\\", "").strip()
+
+
+def build_bibtex(result: AnalysisResult) -> str:
+    """Return a BibTeX string with one ``@inproceedings`` entry per relevant paper."""
+    entries: list[str] = []
+    seen: set[str] = set()
+    for paper in result.relevant_papers:
+        key = base = _cite_key(paper)
+        suffix = ord("a")
+        while key in seen:  # ensure unique keys
+            key = f"{base}{chr(suffix)}"
+            suffix += 1
+        seen.add(key)
+
+        fields = [f"  title = {{{{{_clean(paper.title)}}}}}"]
+        if paper.authors:
+            authors = " and ".join(_clean(a) for a in paper.authors)
+            fields.append(f"  author = {{{authors}}}")
+        year = _year(paper)
+        if year:
+            fields.append(f"  year = {{{year}}}")
+        if paper.url:
+            fields.append(f"  url = {{{paper.url}}}")
+
+        entries.append("@inproceedings{" + key + ",\n" + ",\n".join(fields) + "\n}")
+
+    header = f"% {len(entries)} papers matching \"{_clean(result.theme)}\"\n"
+    return header + "\n\n".join(entries) + ("\n" if entries else "")

conflens/cache.py

@@ -0,0 +1,36 @@
+"""Shared on-disk cache location and a helper to clear it.
+
+The cache holds scraped listings, per-paper abstracts/authors, and classification
+results — all under one directory so it can be wiped in one go.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+from typing import Optional
+
+
+def default_cache_dir() -> str:
+    return os.path.join(os.path.expanduser("~"), ".cache", "conflens")
+
+
+def clear_cache(cache_dir: Optional[str] = None) -> tuple[str, int]:
+    """Delete everything inside the cache directory.
+
+    Returns ``(cache_dir, items_removed)``. The directory itself is kept.
+    """
+    target = cache_dir or default_cache_dir()
+    removed = 0
+    if os.path.isdir(target):
+        for name in os.listdir(target):
+            path = os.path.join(target, name)
+            try:
+                if os.path.isdir(path) and not os.path.islink(path):
+                    shutil.rmtree(path)
+                else:
+                    os.remove(path)
+                removed += 1
+            except OSError:
+                pass
+    return target, removed

conflens/classifier.py

@@ -0,0 +1,174 @@
+"""LLM-based relevance classification of papers against a theme."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from typing import Callable, Optional
+
+from .llm import LLMClient
+from .models import Paper
+
+_BATCH_SIZE = 20
+_ABSTRACT_CHARS = 900  # keep prompts cheap; the title + opening of the abstract is plenty
+
+_SYSTEM = (
+    "You are an expert research librarian for NLP and machine-learning venues. "
+    "You decide whether each paper is genuinely about a given theme. Judge by the "
+    "paper's actual contribution, not incidental keyword matches. Be precise: a paper "
+    "that merely mentions the theme in passing is NOT relevant; a paper whose core "
+    "contribution advances the theme IS relevant."
+)
+
+_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "results": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "index": {"type": "integer"},
+                    "relevant": {"type": "boolean"},
+                    "confidence": {"type": "number"},
+                    "reason": {"type": "string"},
+                },
+                "required": ["index", "relevant", "confidence", "reason"],
+                "additionalProperties": False,
+            },
+        }
+    },
+    "required": ["results"],
+    "additionalProperties": False,
+}
+
+
+def _batch_prompt(theme: str, batch: list[Paper], offset: int, theme_definition: str = "") -> str:
+    lines = [f'Theme: "{theme}"']
+    if theme_definition and theme_definition.strip():
+        lines.append(f"Scope of the theme (what counts / doesn't): {theme_definition.strip()}")
+    lines += [
+        "",
+        "For EACH paper below, decide whether its core contribution is about this "
+        "theme. Return relevant (true/false), a confidence in [0,1], and a one-sentence "
+        "reason. Use the paper's `index` exactly as given.",
+        "",
+    ]
+    for i, p in enumerate(batch):
+        idx = offset + i
+        abstract = (p.abstract or "").strip()
+        if len(abstract) > _ABSTRACT_CHARS:
+            abstract = abstract[:_ABSTRACT_CHARS] + "…"
+        lines.append(f"[{idx}] Title: {p.title}")
+        if abstract:
+            lines.append(f"    Abstract: {abstract}")
+        lines.append("")
+    return "\n".join(lines)
+
+
+def _content_hash(paper: Paper) -> str:
+    raw = f"{paper.title}\n{paper.abstract or ''}"
+    return hashlib.sha1(raw.encode("utf-8")).hexdigest()[:16]
+
+
+def _cache_path(cache_dir: str, cache_sig: str, theme: str, theme_definition: str = "") -> str:
+    digest = hashlib.sha1(
+        f"{cache_sig}|{theme}|{theme_definition}".encode("utf-8")
+    ).hexdigest()[:16]
+    return os.path.join(cache_dir, f"classify_{digest}.json")
+
+
+def classify_papers(
+    client: LLMClient,
+    theme: str,
+    papers: list[Paper],
+    min_confidence: float = 0.5,
+    progress: Optional[Callable[[int, int], None]] = None,
+    cache_dir: Optional[str] = None,
+    cache_sig: str = "",
+    force_refresh: bool = False,
+    cancel: Optional[Callable[[], None]] = None,
+    theme_definition: str = "",
+) -> list[Paper]:
+    """Annotate every paper with relevance/confidence/reason in place.
+
+    The *raw* model judgement (relevant flag, confidence, reason) is cached on
+    disk, keyed by provider+model, theme, and the paper's title+abstract hash —
+    so re-running the same theme/model is free, while changing any of those
+    re-classifies only the affected papers. ``min_confidence`` is applied at read
+    time, so adjusting the threshold never needs a re-call.
+
+    Returns the list of papers judged relevant at or above ``min_confidence``.
+    """
+    total = len(papers)
+
+    # Load any cached judgements for this (provider+model, theme).
+    cache: dict[str, dict] = {}
+    cache_path = None
+    if cache_dir:
+        os.makedirs(cache_dir, exist_ok=True)
+        cache_path = _cache_path(cache_dir, cache_sig, theme, theme_definition)
+        if not force_refresh and os.path.exists(cache_path):
+            try:
+                with open(cache_path, "r", encoding="utf-8") as fh:
+                    cache = json.load(fh)
+            except (OSError, json.JSONDecodeError):
+                cache = {}
+
+    # raw[paper_id] = {"relevant": bool, "confidence": float, "reason": str}
+    raw: dict[str, dict] = {}
+    todo: list[Paper] = []
+    for p in papers:
+        hit = cache.get(p.paper_id)
+        if hit and not force_refresh and hit.get("hash") == _content_hash(p):
+            raw[p.paper_id] = hit
+        else:
+            todo.append(p)
+
+    done = total - len(todo)
+    if progress:
+        progress(done, total)
+
+    def _save() -> None:
+        if cache_path:
+            try:
+                with open(cache_path, "w", encoding="utf-8") as fh:
+                    json.dump(cache, fh)
+            except OSError:
+                pass
+
+    for start in range(0, len(todo), _BATCH_SIZE):
+        if cancel:
+            cancel()
+        batch = todo[start : start + _BATCH_SIZE]
+        prompt = _batch_prompt(theme, batch, start, theme_definition)
+        data = client.structured(_SYSTEM, prompt, _SCHEMA, max_tokens=4000, effort="low")
+        by_index = {r["index"]: r for r in data.get("results", [])}
+        for i, paper in enumerate(batch):
+            r = by_index.get(start + i) or {}
+            entry = {
+                "relevant": bool(r.get("relevant")),
+                "confidence": float(r.get("confidence", 0.0)),
+                "reason": r.get("reason", ""),
+                "hash": _content_hash(paper),
+            }
+            raw[paper.paper_id] = entry
+            cache[paper.paper_id] = entry
+        done += len(batch)
+        if progress:
+            progress(min(done, total), total)
+        _save()  # persist incrementally so a long run is resumable
+
+    _save()
+
+    # Apply the confidence threshold to every paper (cached + freshly judged).
+    relevant: list[Paper] = []
+    for p in papers:
+        entry = raw.get(p.paper_id, {})
+        p.confidence = float(entry.get("confidence", 0.0))
+        p.reason = entry.get("reason", "")
+        p.relevant = bool(entry.get("relevant")) and p.confidence >= min_confidence
+        if p.relevant:
+            relevant.append(p)
+    return relevant

conflens/cli.py

@@ -0,0 +1,75 @@
+"""Console entry point for the Conference Paper Analyzer GUI."""
+
+from __future__ import annotations
+
+import argparse
+
+from nicegui import ui
+
+from .app import create_ui
+from .cache import clear_cache, default_cache_dir
+
+
+@ui.page("/")
+def _index() -> None:
+    create_ui()
+
+
+def _load_env() -> None:
+    """Load a ``.env`` from the current working directory (or above), if present.
+
+    ``find_dotenv(usecwd=True)`` searches from the directory the user runs
+    ``conflens`` in — not the installed package location — so a pip-installed CLI
+    picks up a ``.env`` sitting next to where it's launched. Real environment
+    variables always take precedence (``load_dotenv`` doesn't override them).
+    """
+    try:
+        from dotenv import find_dotenv, load_dotenv
+    except ImportError:  # python-dotenv is a dependency, but stay defensive
+        return
+    path = find_dotenv(usecwd=True)
+    if path:
+        load_dotenv(path)
+
+
+def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        prog="conflens",
+        description="Browse, theme-classify, and topic-model conference papers.",
+    )
+    parser.add_argument(
+        "--clear-cache",
+        action="store_true",
+        help="delete cached scrapes and classification results, then exit",
+    )
+    parser.add_argument(
+        "--cache-dir",
+        default=None,
+        help=f"cache directory (default: {default_cache_dir()})",
+    )
+    parser.add_argument("--host", default="0.0.0.0", help="host to bind (default: 0.0.0.0)")
+    parser.add_argument("--port", type=int, default=6868, help="port to bind (default: 6868)")
+    return parser.parse_args(argv)
+
+
+def main() -> None:
+    """Launch the web UI, or run a maintenance command."""
+    args = _parse_args()
+
+    if args.clear_cache:
+        target, removed = clear_cache(args.cache_dir)
+        print(f"Cleared {removed} item(s) from {target}")
+        return
+
+    _load_env()
+    ui.run(
+        title="Conference Paper Analyzer",
+        host=args.host,
+        port=args.port,
+        reload=False,
+        favicon="🔬",
+    )
+
+
+if __name__ in {"__main__", "__mp_main__"}:
+    main()

conflens/dedup.py

@@ -0,0 +1,100 @@
+"""Lightweight near-duplicate detection over papers, by title similarity.
+
+Dependency-free: exact-normalised-title grouping catches the common case (the
+same paper appearing in several tracks / with reformatted punctuation), plus a
+bounded fuzzy pass (``difflib`` ratio) within first-token buckets to catch minor
+wording differences. No embeddings, so it stays fast and installs nothing.
+
+Each paper in a group of size > 1 (except the representative) gets its
+``duplicate_of`` set to the representative's ``paper_id``.
+"""
+
+from __future__ import annotations
+
+import re
+from difflib import SequenceMatcher
+
+from .models import Paper
+
+_NON_ALNUM = re.compile(r"[^a-z0-9\s]+")
+_WS = re.compile(r"\s+")
+
+_FUZZY_RATIO = 0.92     # normalised-title similarity to treat as a near-duplicate
+_BUCKET_CAP = 400       # skip the fuzzy pass for pathologically large buckets
+
+
+def _normalise(title: str) -> str:
+    return _WS.sub(" ", _NON_ALNUM.sub(" ", title.lower())).strip()
+
+
+class _DSU:
+    def __init__(self, n: int) -> None:
+        self.parent = list(range(n))
+
+    def find(self, x: int) -> int:
+        while self.parent[x] != x:
+            self.parent[x] = self.parent[self.parent[x]]
+            x = self.parent[x]
+        return x
+
+    def union(self, a: int, b: int) -> None:
+        ra, rb = self.find(a), self.find(b)
+        if ra != rb:
+            self.parent[max(ra, rb)] = min(ra, rb)  # keep the earlier paper as root
+
+
+def annotate_duplicates(papers: list[Paper]) -> int:
+    """Set ``duplicate_of`` on near-duplicate papers in place.
+
+    Returns the number of duplicate *groups* (clusters of size > 1).
+    """
+    n = len(papers)
+    for p in papers:            # reset any prior annotation
+        p.duplicate_of = None
+    if n < 2:
+        return 0
+
+    norms = [_normalise(p.title) for p in papers]
+    dsu = _DSU(n)
+
+    # 1. Exact normalised-title matches.
+    exact: dict[str, int] = {}
+    for i, key in enumerate(norms):
+        if not key:
+            continue
+        if key in exact:
+            dsu.union(exact[key], i)
+        else:
+            exact[key] = i
+
+    # 2. Bounded fuzzy pass within first-token buckets.
+    buckets: dict[str, list[int]] = {}
+    for i, key in enumerate(norms):
+        if key:
+            buckets.setdefault(key.split(" ", 1)[0], []).append(i)
+    for members in buckets.values():
+        if len(members) < 2 or len(members) > _BUCKET_CAP:
+            continue
+        for a in range(len(members)):
+            for b in range(a + 1, len(members)):
+                i, j = members[a], members[b]
+                if dsu.find(i) == dsu.find(j):
+                    continue
+                if SequenceMatcher(None, norms[i], norms[j]).ratio() >= _FUZZY_RATIO:
+                    dsu.union(i, j)
+
+    # 3. Collect clusters; annotate non-representatives.
+    groups: dict[int, list[int]] = {}
+    for i in range(n):
+        groups.setdefault(dsu.find(i), []).append(i)
+
+    dup_groups = 0
+    for root, members in groups.items():
+        if len(members) < 2:
+            continue
+        dup_groups += 1
+        rep_id = papers[root].paper_id
+        for idx in members:
+            if idx != root:
+                papers[idx].duplicate_of = rep_id
+    return dup_groups