conflens 0.1.14__py3-none-any.whl

conflens/pptx_export.py

@@ -0,0 +1,225 @@
+"""Render an :class:`AnalysisResult` to a PowerPoint slide deck.
+
+Generated deterministically with ``python-pptx`` (no API call). The styling
+mirrors the web UI: white slides, a single navy accent, clean typography.
+
+Deck structure:
+
+* a **title** slide (theme, source, headline counts);
+* an **overview** slide with a papers-per-topic bar chart; and
+* one or more **topic** slides, each listing its papers with author lines and
+  clickable PDF links (continuation slides are added when a topic overflows).
+"""
+
+from __future__ import annotations
+
+import io
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:  # avoid importing the model at module load
+    from .models import AnalysisResult
+
+# Palette shared with the UI -------------------------------------------------
+_NAVY = (0x1F, 0x4E, 0x79)
+_ACCENT = (0x2B, 0x6C, 0xB0)
+_INK = (0x1A, 0x20, 0x2C)
+_MUTED = (0x64, 0x74, 0x8B)
+_LINE = (0xE2, 0xE8, 0xF0)
+_WHITE = (0xFF, 0xFF, 0xFF)
+
+_PAPERS_PER_SLIDE = 8
+
+
+def build_pptx(result: "AnalysisResult") -> bytes:
+    """Build the deck and return it as ``.pptx`` bytes.
+
+    Raises ``RuntimeError`` with an actionable message if ``python-pptx`` is
+    not installed.
+    """
+    try:
+        from pptx import Presentation
+        from pptx.chart.data import CategoryChartData
+        from pptx.dml.color import RGBColor
+        from pptx.enum.chart import XL_CHART_TYPE
+        from pptx.enum.text import MSO_ANCHOR
+        from pptx.util import Inches, Pt
+    except ImportError as e:  # pragma: no cover - optional dependency
+        raise RuntimeError(
+            "PPTX export needs the 'python-pptx' package. Install it with "
+            "`pip install python-pptx`."
+        ) from e
+
+    def rgb(c: tuple[int, int, int]) -> "RGBColor":
+        return RGBColor(*c)
+
+    prs = Presentation()
+    prs.slide_width = Inches(13.333)
+    prs.slide_height = Inches(7.5)
+    blank = prs.slide_layouts[6]
+    SW, SH = prs.slide_width, prs.slide_height
+
+    # ---- small helpers ------------------------------------------------ #
+    def add_slide():
+        return prs.slides.add_slide(blank)
+
+    def fill_rect(slide, left, top, width, height, color):
+        from pptx.enum.shapes import MSO_SHAPE
+
+        shape = slide.shapes.add_shape(MSO_SHAPE.RECTANGLE, left, top, width, height)
+        shape.fill.solid()
+        shape.fill.fore_color.rgb = rgb(color)
+        shape.line.fill.background()
+        shape.shadow.inherit = False
+        return shape
+
+    def textbox(slide, left, top, width, height):
+        box = slide.shapes.add_textbox(left, top, width, height)
+        tf = box.text_frame
+        tf.word_wrap = True
+        return box, tf
+
+    def style_run(run, text, size, color, bold=False, italic=False):
+        run.text = text
+        run.font.size = Pt(size)
+        run.font.bold = bold
+        run.font.italic = italic
+        run.font.color.rgb = rgb(color)
+        run.font.name = "Calibri"
+
+    def header_bar(slide, title, subtitle=""):
+        fill_rect(slide, 0, 0, SW, Inches(1.0), _NAVY)
+        _, tf = textbox(slide, Inches(0.5), Inches(0.12), SW - Inches(1.0), Inches(0.8))
+        tf.vertical_anchor = MSO_ANCHOR.MIDDLE
+        p = tf.paragraphs[0]
+        style_run(p.add_run(), title, 24, _WHITE, bold=True)
+        if subtitle:
+            p2 = tf.add_paragraph()
+            style_run(p2.add_run(), subtitle, 12, _WHITE)
+
+    # ---- 1. title slide ---------------------------------------------- #
+    s = add_slide()
+    fill_rect(s, 0, 0, SW, SH, _WHITE)
+    fill_rect(s, 0, Inches(2.4), SW, Inches(0.06), _ACCENT)
+
+    _, tf = textbox(s, Inches(0.9), Inches(1.2), SW - Inches(1.8), Inches(1.2))
+    style_run(tf.paragraphs[0].add_run(), "Conference Paper Analysis", 20, _MUTED)
+    p = tf.add_paragraph()
+    style_run(p.add_run(), result.theme, 40, _NAVY, bold=True)
+
+    _, tf = textbox(s, Inches(0.9), Inches(2.7), SW - Inches(1.8), Inches(3.0))
+    line = tf.paragraphs[0]
+    style_run(line.add_run(), f"Source: {result.event_url}", 14, _MUTED)
+    stats = [
+        (str(result.scanned), "papers scanned"),
+        (str(len(result.relevant_papers)), f"match “{result.theme}”"),
+        (str(len(result.topics)), "topics discovered"),
+    ]
+    for value, label in stats:
+        p = tf.add_paragraph()
+        p.space_before = Pt(10)
+        style_run(p.add_run(), f"{value}  ", 28, _ACCENT, bold=True)
+        style_run(p.add_run(), label, 16, _INK)
+
+    # ---- 2. overview / chart ----------------------------------------- #
+    if result.topics:
+        s = add_slide()
+        fill_rect(s, 0, 0, SW, SH, _WHITE)
+        header_bar(s, "Papers per topic", result.theme)
+        chart_data = CategoryChartData()
+        # reversed so the largest topic sits at the top of a horizontal bar chart
+        topics_sorted = list(result.topics)
+        chart_data.categories = [t.name for t in reversed(topics_sorted)]
+        chart_data.add_series("Papers", [t.count for t in reversed(topics_sorted)])
+        gframe = s.shapes.add_chart(
+            XL_CHART_TYPE.BAR_CLUSTERED,
+            Inches(0.5), Inches(1.2), SW - Inches(1.0), SH - Inches(1.6),
+            chart_data,
+        )
+        chart = gframe.chart
+        chart.has_legend = False
+        chart.has_title = False
+        plot = chart.plots[0]
+        plot.has_data_labels = True
+        plot.data_labels.font.size = Pt(11)
+        plot.data_labels.font.color.rgb = rgb(_INK)
+        plot.gap_width = 60
+        series = chart.series[0]
+        series.format.fill.solid()
+        series.format.fill.fore_color.rgb = rgb(_ACCENT)
+        for axis in (chart.category_axis, chart.value_axis):
+            axis.tick_labels.font.size = Pt(11)
+            axis.tick_labels.font.color.rgb = rgb(_INK)
+
+    # ---- 3. one (or more) slide(s) per topic ------------------------- #
+    by_id = {p.paper_id: p for p in result.relevant_papers}
+    for topic in result.topics:
+        papers = [by_id[pid] for pid in topic.paper_ids if pid in by_id]
+        papers.sort(key=lambda p: p.confidence or 0, reverse=True)
+
+        # 3a. topic overview slide: description + common findings
+        if topic.description or topic.findings:
+            s = add_slide()
+            fill_rect(s, 0, 0, SW, SH, _WHITE)
+            header_bar(
+                s,
+                topic.name,
+                f"{topic.count} paper{'s' if topic.count != 1 else ''}",
+            )
+            top = Inches(1.25)
+            if topic.description:
+                _, tf = textbox(s, Inches(0.5), top, SW - Inches(1.0), Inches(0.9))
+                style_run(tf.paragraphs[0].add_run(), topic.description, 14, _INK)
+                top = Inches(2.05)
+            if topic.findings:
+                _, tf = textbox(s, Inches(0.5), top, SW - Inches(1.0), SH - top - Inches(0.4))
+                head = tf.paragraphs[0]
+                style_run(head.add_run(), "Main findings across this topic", 12, _ACCENT, bold=True)
+                for finding in topic.findings[:10]:
+                    fp = tf.add_paragraph()
+                    fp.space_before = Pt(5)
+                    style_run(fp.add_run(), "•  " + finding, 13, _INK)
+        chunks = [
+            papers[i : i + _PAPERS_PER_SLIDE]
+            for i in range(0, max(len(papers), 1), _PAPERS_PER_SLIDE)
+        ] or [[]]
+        for idx, chunk in enumerate(chunks):
+            s = add_slide()
+            fill_rect(s, 0, 0, SW, SH, _WHITE)
+            suffix = "" if len(chunks) == 1 else f"  ({idx + 1}/{len(chunks)})"
+            header_bar(
+                s,
+                f"{topic.name} — Papers{suffix}",
+                f"{topic.count} paper{'s' if topic.count != 1 else ''}",
+            )
+            top = Inches(1.25)
+            _, tf = textbox(s, Inches(0.5), top, SW - Inches(1.0), SH - top - Inches(0.3))
+            first = True
+            for paper in chunk:
+                p = tf.paragraphs[0] if first else tf.add_paragraph()
+                first = False
+                p.space_after = Pt(8)
+                # title (clickable -> landing page when available)
+                trun = p.add_run()
+                style_run(trun, "• " + paper.title, 14, _NAVY, bold=True)
+                if paper.url:
+                    trun.hyperlink.address = paper.url
+                if paper.confidence is not None:
+                    style_run(p.add_run(), f"   ({paper.confidence:.0%})", 11, _MUTED)
+                # PDF link (only when a PDF exists)
+                if paper.pdf_url:
+                    style_run(p.add_run(), "   ", 14, _INK)
+                    pdf = p.add_run()
+                    style_run(pdf, "[PDF]", 12, _ACCENT, bold=True)
+                    pdf.hyperlink.address = paper.pdf_url
+                # authors on a second line
+                if paper.authors:
+                    authors = ", ".join(paper.authors[:6]) + (
+                        "…" if len(paper.authors) > 6 else ""
+                    )
+                    a = tf.add_paragraph()
+                    a.space_after = Pt(8)
+                    style_run(a.add_run(), authors, 11, _MUTED)
+
+    bio = io.BytesIO()
+    prs.save(bio)
+    return bio.getvalue()

conflens/scraper.py

@@ -0,0 +1,277 @@
+"""Scraping logic for the ACL Anthology (and structurally compatible sites).
+
+The Anthology is a static Hugo site with minified, mostly-unquoted HTML, so we
+parse with targeted regexes rather than a full DOM parser to keep dependencies
+light and behaviour predictable.
+
+Two levels of pages matter:
+
+* an *event* page (e.g. ``/events/acl-2024/``) lists every paper with its
+  title, landing-page URL and PDF link, but **not** its abstract; and
+* a *paper* page (e.g. ``/2024.acl-long.1/``) carries the abstract plus clean
+  author metadata in ``<meta name="citation_author">`` tags.
+
+Abstracts are fetched lazily and cached on disk so re-runs are cheap.
+"""
+
+from __future__ import annotations
+
+import concurrent.futures
+import hashlib
+import html
+import http.client
+import json
+import os
+import re
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from typing import Callable, Optional
+
+from .models import Paper
+
+_UA = "conflens/0.1 (+https://github.com/picaultj/conflens)"
+_DEFAULT_BASE = "https://aclanthology.org"
+
+# A paper landing-page anchor, e.g.  <a ... href=/2024.acl-long.1/>Title</a>
+_PAPER_ANCHOR = re.compile(
+    r'<a[^>]*\bhref=(?P<q>["\']?)(?P<path>/(?P<id>\d{4}\.[a-z0-9]+-[a-z0-9]+\.\d+)/)(?P=q)[^>]*>'
+    r"(?P<title>.*?)</a>",
+    re.IGNORECASE | re.DOTALL,
+)
+
+_ABSTRACT = re.compile(
+    r'<div[^>]*class="[^"]*acl-abstract[^"]*"[^>]*>\s*'
+    r"(?:<h[0-9][^>]*>.*?</h[0-9]>)?\s*<span>(?P<abs>.*?)</span>",
+    re.IGNORECASE | re.DOTALL,
+)
+
+# Author meta tags appear as `<meta content="Name" name=citation_author>` — note
+# the content-before-name order and unquoted attribute names in the minified HTML.
+_META_TAG = re.compile(r"<meta\b[^>]*>", re.IGNORECASE)
+_META_CONTENT = re.compile(r'\bcontent=["\'](?P<v>[^"\']*)["\']', re.IGNORECASE)
+_META_NAME = re.compile(r'\bname=["\']?citation_author["\']?', re.IGNORECASE)
+
+
+def _extract_authors(page: str) -> list[str]:
+    authors: list[str] = []
+    for tag in _META_TAG.findall(page):
+        if _META_NAME.search(tag):
+            m = _META_CONTENT.search(tag)
+            if m:
+                authors.append(html.unescape(m.group("v")))
+    return authors
+
+_TAG = re.compile(r"<[^>]+>")
+_WS = re.compile(r"\s+")
+
+
+def _clean(text: str) -> str:
+    """Strip tags, unescape entities and collapse whitespace."""
+    return _WS.sub(" ", html.unescape(_TAG.sub("", text))).strip()
+
+
+class AnthologyScraper:
+    """Fetches and parses paper listings + abstracts."""
+
+    def __init__(
+        self,
+        base_url: str = _DEFAULT_BASE,
+        cache_dir: Optional[str] = None,
+        timeout: int = 60,
+        max_workers: int = 8,
+    ) -> None:
+        self.base_url = base_url.rstrip("/")
+        self.timeout = timeout
+        self.max_workers = max_workers
+        self.cache_dir = cache_dir or os.path.join(
+            os.path.expanduser("~"), ".cache", "conflens"
+        )
+        os.makedirs(self.cache_dir, exist_ok=True)
+
+    # ------------------------------------------------------------------ #
+    # HTTP helpers
+    # ------------------------------------------------------------------ #
+    def _get(self, url: str) -> str:
+        req = urllib.request.Request(url, headers={"User-Agent": _UA})
+        last_err: Optional[Exception] = None
+        partial_best = ""
+        for attempt in range(4):
+            try:
+                with urllib.request.urlopen(req, timeout=self.timeout) as resp:
+                    return resp.read().decode("utf-8", "replace")
+            except http.client.IncompleteRead as e:
+                # Large listing pages occasionally truncate. Retry for a complete
+                # read so we don't cache a partial listing; keep the longest
+                # partial seen as a last-resort fallback.
+                chunk = e.partial.decode("utf-8", "replace")
+                if len(chunk) > len(partial_best):
+                    partial_best = chunk
+                last_err = e
+                time.sleep(1.0 * (attempt + 1))
+            except (urllib.error.URLError, TimeoutError) as e:
+                last_err = e
+                time.sleep(1.5 * (attempt + 1))
+        if partial_best:
+            return partial_best
+        raise RuntimeError(f"Failed to fetch {url}: {last_err}")
+
+    def resolve_url(self, target: str) -> str:
+        """Unified source interface — alias of :meth:`event_url`."""
+        return self.event_url(target)
+
+    def event_url(self, event: str) -> str:
+        """Build an event URL from a slug, or pass through a full URL."""
+        event = event.strip()
+        if event.startswith("http://") or event.startswith("https://"):
+            return event
+        event = event.strip("/")
+        if event.startswith("events/"):
+            return f"{self.base_url}/{event}/"
+        return f"{self.base_url}/events/{event}/"
+
+    # ------------------------------------------------------------------ #
+    # Listing
+    # ------------------------------------------------------------------ #
+    def _listing_cache_path(self, key: str) -> str:
+        digest = hashlib.sha1(key.encode("utf-8")).hexdigest()[:16]
+        return os.path.join(self.cache_dir, f"listing_{digest}.json")
+
+    def list_papers(
+        self,
+        event: str,
+        include_frontmatter: bool = False,
+        force_refresh: bool = False,
+    ) -> list[Paper]:
+        """Return every paper listed on an event page (without abstracts).
+
+        The parsed listing is cached on disk and keyed by event URL, so running
+        several analyses (e.g. for different themes) against the same event does
+        not re-download the multi-megabyte listing page. Pass
+        ``force_refresh=True`` to bypass and rebuild the cache.
+        """
+        url = self.event_url(event)
+        cache = self._listing_cache_path(url + ("|fm" if include_frontmatter else ""))
+        if not force_refresh and os.path.exists(cache):
+            try:
+                with open(cache, "r", encoding="utf-8") as fh:
+                    data = json.load(fh)
+                return [
+                    Paper(
+                        paper_id=p["paper_id"],
+                        title=p["title"],
+                        url=p["url"],
+                        pdf_url=p["pdf_url"],
+                    )
+                    for p in data.get("papers", [])
+                ]
+            except (OSError, json.JSONDecodeError, KeyError, TypeError):
+                pass  # corrupt/legacy cache — fall through and refetch
+
+        page = self._get(url)
+        papers = self.parse_listing(page, include_frontmatter=include_frontmatter)
+        try:
+            with open(cache, "w", encoding="utf-8") as fh:
+                json.dump(
+                    {
+                        "event_url": url,
+                        "papers": [
+                            {
+                                "paper_id": p.paper_id,
+                                "title": p.title,
+                                "url": p.url,
+                                "pdf_url": p.pdf_url,
+                            }
+                            for p in papers
+                        ],
+                    },
+                    fh,
+                )
+        except OSError:
+            pass
+        return papers
+
+    def parse_listing(self, page: str, include_frontmatter: bool = False) -> list[Paper]:
+        """Parse an event listing page's HTML into papers (no network)."""
+        seen: set[str] = set()
+        papers: list[Paper] = []
+        for m in _PAPER_ANCHOR.finditer(page):
+            pid = m.group("id")
+            if pid in seen:
+                continue
+            # ``*.0`` entries are the proceedings front-matter, not real papers.
+            if pid.rsplit(".", 1)[-1] == "0" and not include_frontmatter:
+                continue
+            title = _clean(m.group("title"))
+            if not title:
+                continue
+            seen.add(pid)
+            papers.append(
+                Paper(
+                    paper_id=pid,
+                    title=title,
+                    url=f"{self.base_url}/{pid}/",
+                    pdf_url=f"{self.base_url}/{pid}.pdf",
+                )
+            )
+        return papers
+
+    # ------------------------------------------------------------------ #
+    # Abstracts / metadata
+    # ------------------------------------------------------------------ #
+    @staticmethod
+    def parse_detail(page: str) -> tuple[str, list[str]]:
+        """Parse a paper page's HTML into ``(abstract, authors)`` (no network)."""
+        m = _ABSTRACT.search(page)
+        abstract = _clean(m.group("abs")) if m else ""
+        return abstract, _extract_authors(page)
+
+    def _abstract_cache_path(self, pid: str) -> str:
+        safe = pid.replace("/", "_")
+        return os.path.join(self.cache_dir, f"{safe}.json")
+
+    def _fetch_detail(self, paper: Paper, force_refresh: bool = False) -> Paper:
+        cache = self._abstract_cache_path(paper.paper_id)
+        if not force_refresh and os.path.exists(cache):
+            try:
+                with open(cache, "r", encoding="utf-8") as fh:
+                    data = json.load(fh)
+                paper.abstract = data.get("abstract", "")
+                paper.authors = data.get("authors", [])
+                return paper
+            except (OSError, json.JSONDecodeError):
+                pass
+        try:
+            page = self._get(paper.url)
+        except RuntimeError:
+            return paper
+        paper.abstract, paper.authors = self.parse_detail(page)
+        try:
+            with open(cache, "w", encoding="utf-8") as fh:
+                json.dump({"abstract": paper.abstract, "authors": paper.authors}, fh)
+        except OSError:
+            pass
+        return paper
+
+    def enrich_abstracts(
+        self,
+        papers: list[Paper],
+        progress: Optional[Callable[[int, int], None]] = None,
+        force_refresh: bool = False,
+    ) -> list[Paper]:
+        """Fetch abstracts + authors for ``papers`` concurrently (disk-cached)."""
+        total = len(papers)
+        done = 0
+        if progress:
+            progress(0, total)
+        with concurrent.futures.ThreadPoolExecutor(max_workers=self.max_workers) as ex:
+            futures = {
+                ex.submit(self._fetch_detail, p, force_refresh): p for p in papers
+            }
+            for fut in concurrent.futures.as_completed(futures):
+                fut.result()
+                done += 1
+                if progress:
+                    progress(done, total)
+        return papers