superlab 0.1.64 → 0.1.66

package/package-assets/shared/lab/.managed/scripts/extract_reference_paper_structure.py

@@ -0,0 +1,1200 @@
+#!/usr/bin/env python3
+"""Extract reusable writing templates from reference papers.
+
+This script is intentionally lightweight. It extracts structure, paragraph
+roles, and visual/table roles for `/lab:write`; it does not summarize or copy
+paper content.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import html
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import urllib.parse
+import urllib.request
+from dataclasses import asdict, dataclass
+from pathlib import Path
+
+
+SECTION_ALIASES = {
+    "abstract": ("abstract", "摘要"),
+    "introduction": ("introduction", "intro", "引言", "绪论"),
+    "related-work": ("related work", "background", "literature", "相关工作", "文献综述"),
+    "method": (
+        "method",
+        "methods",
+        "methodology",
+        "approach",
+        "model",
+        "algorithm",
+        "方法",
+        "模型",
+        "算法",
+    ),
+    "experiments": (
+        "experiment",
+        "experiments",
+        "evaluation",
+        "empirical",
+        "results",
+        "main results",
+        "ablation",
+        "sensitivity",
+        "coverage",
+        "实验",
+        "评估",
+        "结果",
+    ),
+    "discussion": ("discussion", "analysis", "讨论", "分析"),
+    "conclusion": ("conclusion", "conclusions", "future work", "结论", "总结"),
+    "references": ("references", "bibliography", "参考文献"),
+    "appendix": ("appendix", "supplement", "附录"),
+}
+
+CANONICAL_HEADING_TITLES = {
+    "abstract",
+    "introduction",
+    "related work",
+    "background",
+    "method",
+    "methods",
+    "methodology",
+    "approach",
+    "model",
+    "algorithm",
+    "experiments",
+    "experiment",
+    "experimental setup",
+    "experimental setups",
+    "evaluation",
+    "results",
+    "main results",
+    "ablation study",
+    "sensitivity analysis",
+    "discussion",
+    "conclusion",
+    "conclusions",
+    "references",
+    "bibliography",
+    "appendix",
+}
+
+EXPERIMENT_PROTOCOL_SLOT_GUIDANCE = {
+    "dataset_description": {
+        "reader_question": "Which datasets define the evaluation scope, and why are they relevant?",
+        "placement_guidance": "place before baselines, metrics, and main results",
+    },
+    "dataset_statistics": {
+        "reader_question": "What dataset scale, feature, treatment, or split facts constrain interpretation?",
+        "placement_guidance": "place near dataset descriptions or move detailed statistics to appendix with a main-text pointer",
+    },
+    "split_protocol": {
+        "reader_question": "How are train, validation, test, seed, or sampling decisions made?",
+        "placement_guidance": "place before metrics and results so comparisons have a fixed protocol",
+    },
+    "baseline_setup": {
+        "reader_question": "Which comparator families are included, and what role does each comparator play?",
+        "placement_guidance": "place after datasets and before the main comparison table",
+    },
+    "metric_definition": {
+        "reader_question": "Which metrics decide ranking, what do they measure, and which direction is better?",
+        "placement_guidance": "place before the first result table and repeat local definitions in table notes when needed",
+    },
+    "implementation_details": {
+        "reader_question": "Which tuning, validation, training, or hardware details are needed for reproducibility?",
+        "placement_guidance": "place after metrics or in appendix when details are long",
+    },
+    "main_results": {
+        "reader_question": "What is the primary comparison result under the declared protocol?",
+        "placement_guidance": "place after setup, baselines, and metrics",
+    },
+    "ablation": {
+        "reader_question": "Which component or design choice accounts for the claimed effect?",
+        "placement_guidance": "place after the main results",
+    },
+    "sensitivity": {
+        "reader_question": "How stable is the result under relevant protocol or hyperparameter changes?",
+        "placement_guidance": "place after ablations or in an analysis subsection",
+    },
+    "appendix_dataset_statistics": {
+        "reader_question": "Which detailed dataset facts support the compact dataset setup in the main experiments?",
+        "placement_guidance": "link from the main experimental setup and keep detailed tables in appendix",
+    },
+    "appendix_baseline_metric_details": {
+        "reader_question": "Which baseline, metric, or implementation details are too long for the main setup?",
+        "placement_guidance": "link from the main setup and keep long comparator or metric definitions in appendix",
+    },
+}
+
+
+@dataclass
+class SectionRecord:
+    title: str
+    section_type: str
+    level: int
+    start_line: int
+    end_line: int
+    start_page: int | None
+    end_page: int | None
+    content: str
+
+
+@dataclass
+class AssetRecord:
+    asset_type: str
+    asset_id: str
+    caption: str
+    appears_in_section: str
+    appears_in_title: str
+    evidence_role: str
+    local_role: str
+    placement_logic: str
+    text_bridge_before: str
+    text_bridge_after: str
+    reuse_guidance: str
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Extract reference-paper structure templates for /lab:write."
+    )
+    parser.add_argument("sources", nargs="+", help="Local paper paths or http(s) PDF/HTML URLs")
+    parser.add_argument(
+        "--output-dir",
+        required=True,
+        help="Directory to write reference-pattern artifacts",
+    )
+    return parser.parse_args()
+
+
+def slugify(value: str) -> str:
+    value = re.sub(r"\.[A-Za-z0-9]+$", "", value)
+    slug = re.sub(r"[^A-Za-z0-9]+", "-", value).strip("-").lower()
+    return slug[:80] or "reference-paper"
+
+
+def short_excerpt(text: str, limit: int = 240) -> str:
+    cleaned = re.sub(r"\s+", " ", text).strip()
+    if len(cleaned) <= limit:
+        return cleaned
+    return cleaned[: limit - 3].rstrip() + "..."
+
+
+def materialize_source(source: str, cache_dir: Path) -> tuple[Path, str]:
+    parsed = urllib.parse.urlparse(source)
+    if parsed.scheme in {"http", "https"}:
+        cache_dir.mkdir(parents=True, exist_ok=True)
+        suffix = Path(parsed.path).suffix or ".pdf"
+        name = slugify(Path(parsed.path).name or parsed.netloc)
+        digest = hashlib.sha256(source.encode("utf-8")).hexdigest()[:10]
+        target = cache_dir / f"{name}-{digest}{suffix}"
+        urllib.request.urlretrieve(source, target)
+        return target, "url"
+    return Path(source).expanduser().resolve(), "local"
+
+
+def extract_pdf_text(path: Path) -> tuple[str, str, int | None]:
+    errors: list[str] = []
+
+    try:
+        from pypdf import PdfReader  # type: ignore
+
+        reader = PdfReader(str(path))
+        page_texts = []
+        for index, page in enumerate(reader.pages, start=1):
+            page_texts.append(f"[Page {index}]\n{page.extract_text() or ''}")
+        return "\n\n".join(page_texts), "pypdf", len(reader.pages)
+    except Exception as exc:
+        errors.append(f"pypdf: {exc}")
+
+    try:
+        import pdfplumber  # type: ignore
+
+        page_texts = []
+        with pdfplumber.open(str(path)) as pdf:
+            for index, page in enumerate(pdf.pages, start=1):
+                page_texts.append(f"[Page {index}]\n{page.extract_text() or ''}")
+            return "\n\n".join(page_texts), "pdfplumber", len(pdf.pages)
+    except Exception as exc:
+        errors.append(f"pdfplumber: {exc}")
+
+    try:
+        import fitz  # type: ignore
+
+        doc = fitz.open(str(path))
+        page_texts = [f"[Page {i + 1}]\n{page.get_text()}" for i, page in enumerate(doc)]
+        page_count = len(doc)
+        doc.close()
+        return "\n\n".join(page_texts), "pymupdf", page_count
+    except Exception as exc:
+        errors.append(f"pymupdf: {exc}")
+
+    if shutil.which("pdftotext"):
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            out_path = Path(tmp_dir) / "paper.txt"
+            command = ["pdftotext", "-layout", str(path), str(out_path)]
+            completed = subprocess.run(command, capture_output=True, text=True, check=False)
+            if completed.returncode == 0:
+                return out_path.read_text(encoding="utf-8", errors="ignore"), "pdftotext", None
+            errors.append(f"pdftotext: {completed.stderr.strip()}")
+
+    raise RuntimeError(
+        "No PDF text extractor succeeded. Install pypdf, pdfplumber, PyMuPDF, or pdftotext. "
+        + " | ".join(errors)
+    )
+
+
+def find_uv() -> str | None:
+    uv_path = shutil.which("uv")
+    if uv_path:
+        return uv_path
+    local_uv = Path.home() / ".local" / "bin" / "uv"
+    if local_uv.exists():
+        return str(local_uv)
+    return None
+
+
+def rerun_with_uv_pypdf() -> int | None:
+    if os.environ.get("LAB_REFERENCE_PDF_BOOTSTRAPPED"):
+        return None
+    uv_path = find_uv()
+    if not uv_path:
+        return None
+    env = os.environ.copy()
+    env["LAB_REFERENCE_PDF_BOOTSTRAPPED"] = "1"
+    command = [
+        uv_path,
+        "run",
+        "--with",
+        "pypdf",
+        "python3",
+        str(Path(__file__).resolve()),
+        *sys.argv[1:],
+    ]
+    return subprocess.run(command, env=env, check=False).returncode
+
+
+def strip_html(text: str) -> str:
+    text = re.sub(r"(?is)<(script|style).*?</\1>", " ", text)
+    text = re.sub(r"(?i)<br\s*/?>", "\n", text)
+    text = re.sub(r"(?i)</(p|div|section|article|h[1-6]|li|tr)>", "\n", text)
+    text = re.sub(r"<[^>]+>", " ", text)
+    return html.unescape(re.sub(r"\n{3,}", "\n\n", text))
+
+
+def read_source_text(path: Path) -> tuple[str, str, int | None]:
+    suffix = path.suffix.lower()
+    if suffix == ".pdf":
+        return extract_pdf_text(path)
+    raw = path.read_text(encoding="utf-8", errors="ignore")
+    if suffix in {".html", ".htm"}:
+        return strip_html(raw), "html-text", None
+    return raw, "plain-text", None
+
+
+def detect_title(text: str, fallback: str) -> str:
+    for line in text.splitlines():
+        stripped = re.sub(r"^#+\s*", "", line).strip()
+        if not stripped or stripped.startswith("[Page "):
+            continue
+        if is_caption_line(stripped):
+            continue
+        if len(stripped) < 180:
+            return stripped
+    return fallback
+
+
+def normalize_heading(line: str) -> str:
+    line = re.sub(r"^#+\s*", "", line.strip())
+    return line.strip()
+
+
+def section_type_for_title(title: str) -> str:
+    lowered = title.lower()
+    if re.match(r"^(appendix\b|[a-z]\.\d+(?:\.\d+)*\.?\s+)", lowered.strip()):
+        return "appendix"
+    lowered = re.sub(r"^\d+(?:\.\d+)*\.?\s+", "", lowered).strip()
+    for section_type, aliases in SECTION_ALIASES.items():
+        if any(alias in lowered for alias in aliases):
+            return section_type
+    return "other"
+
+
+def heading_level(title: str) -> int:
+    markdown = re.match(r"^(#{1,6})\s+", title)
+    if markdown:
+        return len(markdown.group(1))
+    numeric = re.match(r"^(\d+(?:\.\d+)*)\.?\s+", title.strip())
+    if numeric:
+        return min(1 + numeric.group(1).count("."), 4)
+    return 1
+
+
+def is_caption_line(line: str) -> bool:
+    return bool(re.match(r"^(?:Table|TABLE|Figure|FIGURE|Fig\.?)\s+[A-Za-z0-9.]+", line.strip()))
+
+
+def is_heading_line(line: str) -> bool:
+    stripped = line.strip()
+    if not stripped or stripped.startswith("[Page ") or is_caption_line(stripped):
+        return False
+    if len(stripped) > 140:
+        return False
+    if stripped.endswith((".", ",", ";", ":")) and not re.match(r"^\d+(?:\.\d+)*\.?\s+", stripped):
+        return False
+    if re.match(r"^#{1,4}\s+\S", stripped):
+        return True
+    lowered = re.sub(r"^\d+(?:\.\d+)*\.?\s+", "", stripped.lower()).strip()
+    lowered = re.sub(r"^[ivx]+\.\s+", "", lowered).strip()
+    lowered = re.sub(r"^appendix\s+[a-z](?:\.\d+)*\.?\s*", "", lowered).strip()
+    if lowered in CANONICAL_HEADING_TITLES:
+        return True
+    if re.match(r"^\d+(?:\.\d+)*\.?\s+[A-Z][A-Za-z0-9 ,:;()/-]{2,}$", stripped):
+        return True
+    if re.match(r"^[A-Z]\.\d+(?:\.\d+)*\.?\s+[A-Z][A-Za-z0-9 ,:;()/-]{2,}$", stripped):
+        return True
+    if re.match(r"^Appendix\s+[A-Z](?:\.\d+)*\.?\s*[:\-]?\s*[A-Z][A-Za-z0-9 ,:;()/-]{2,}$", stripped):
+        return True
+    if re.match(r"^[IVX]+\.\s+[A-Z][A-Za-z0-9 ,:;()/-]{2,}$", stripped):
+        return True
+    return False
+
+
+def split_sections(text: str) -> list[SectionRecord]:
+    lines = text.splitlines()
+    headings: list[tuple[int, str, int | None]] = []
+    current_page: int | None = None
+    line_pages: dict[int, int | None] = {}
+
+    for index, line in enumerate(lines):
+        page_match = re.match(r"^\[Page\s+(\d+)\]$", line.strip())
+        if page_match:
+            current_page = int(page_match.group(1))
+        line_pages[index] = current_page
+        if is_heading_line(line):
+            headings.append((index, normalize_heading(line), current_page))
+
+    if not headings:
+        content = "\n".join(line for line in lines if not line.startswith("[Page "))
+        return [
+            SectionRecord(
+                title="Full Text",
+                section_type="other",
+                level=1,
+                start_line=1,
+                end_line=len(lines),
+                start_page=None,
+                end_page=None,
+                content=content,
+            )
+        ]
+
+    sections: list[SectionRecord] = []
+    for pos, (start_index, title, start_page) in enumerate(headings):
+        end_index = headings[pos + 1][0] - 1 if pos + 1 < len(headings) else len(lines) - 1
+        content_lines = [
+            line for line in lines[start_index + 1 : end_index + 1] if not line.startswith("[Page ")
+        ]
+        end_page = line_pages.get(end_index)
+        sections.append(
+            SectionRecord(
+                title=title,
+                section_type=section_type_for_title(title),
+                level=heading_level(title),
+                start_line=start_index + 1,
+                end_line=end_index + 1,
+                start_page=start_page,
+                end_page=end_page,
+                content="\n".join(content_lines).strip(),
+            )
+        )
+    infer_section_types(sections)
+    return sections
+
+
+def infer_section_types(sections: list[SectionRecord]) -> None:
+    """Recover section families when PDF text only preserves numbered headings.
+
+    Many academic PDF extractors keep subsection titles such as "4.2 Feature
+    Selection" but lose the visual hierarchy that tells us they belong to the
+    Method section. For template extraction, section family is more useful than
+    the literal heading string, so unknown subsections inherit from their nearest
+    known parent. Unknown top-level technical sections before Experiments are
+    treated as method-style sections, which matches common ML paper structure.
+    """
+
+    first_experiment_index = next(
+        (index for index, section in enumerate(sections) if section.section_type == "experiments"),
+        None,
+    )
+    first_intro_index = next(
+        (index for index, section in enumerate(sections) if section.section_type == "introduction"),
+        None,
+    )
+    parent_by_level: dict[int, str] = {}
+    inheritable = {"related-work", "method", "experiments", "discussion", "conclusion", "appendix"}
+
+    for index, section in enumerate(sections):
+        for level in list(parent_by_level):
+            if level >= section.level:
+                parent_by_level.pop(level, None)
+
+        if section.section_type == "other":
+            inherited = None
+            for level in range(section.level - 1, 0, -1):
+                parent_type = parent_by_level.get(level)
+                if parent_type in inheritable:
+                    inherited = parent_type
+                    break
+            if inherited:
+                section.section_type = inherited
+            elif (
+                section.level == 1
+                and first_experiment_index is not None
+                and index < first_experiment_index
+                and (first_intro_index is None or index > first_intro_index)
+            ):
+                section.section_type = "method"
+
+        parent_by_level[section.level] = section.section_type
+
+
+def paragraph_role(section_type: str, paragraph: str) -> str:
+    lowered = paragraph.lower()
+    if is_caption_line(paragraph):
+        return "visual_or_table_anchor"
+    if section_type == "abstract":
+        return "abstract_summary"
+    if any(token in lowered for token in ("dataset statistics", "statistics of the dataset", "sample count", "feature dimension")):
+        return "dataset_statistics"
+    if re.search(r"\bdatasets?\s*[:.]", lowered) or any(
+        token in lowered for token in ("benchmark dataset", "public dataset", "semi-synthetic dataset")
+    ):
+        return "dataset_description"
+    if re.search(r"\bbaselines?\s*[:.]", lowered) or any(
+        token in lowered for token in ("baseline family", "comparator", "compare with")
+    ):
+        return "baseline_setup"
+    if re.search(r"\bmetrics?\s*[:.]", lowered) or any(
+        token in lowered
+        for token in (
+            "metric definition",
+            "we report",
+            "primary metric",
+            "secondary metric",
+            "higher is better",
+            "lower is better",
+            "auuc",
+            "qini",
+        )
+    ):
+        return "metric_definition"
+    if any(
+        token in lowered
+        for token in (
+            "train/test",
+            "train, validation",
+            "training split",
+            "validation split",
+            "test split",
+            "random split",
+            "repeated split",
+            "sampling",
+            "seed",
+            "protocol",
+        )
+    ):
+        return "split_protocol"
+    if any(
+        token in lowered
+        for token in (
+            "implementation detail",
+            "hyperparameter",
+            "tuning",
+            "learning rate",
+            "epoch",
+            "batch size",
+            "hardware",
+            "gpu",
+        )
+    ):
+        return "implementation_details"
+    if any(token in lowered for token in ("limitation", "future work", "caveat", "drift", "局限")):
+        return "limitation_boundary"
+    if any(token in lowered for token in ("ablation", "component", "without", "remove")):
+        return "ablation_interpretation"
+    if any(token in lowered for token in ("result", "outperform", "improve", "coverage", "interval length", "gain")):
+        return "result_interpretation"
+    if any(token in lowered for token in ("baseline", "compare", "comparator")):
+        return "comparator_setup"
+    if any(token in lowered for token in ("dataset", "benchmark", "metric", "protocol", "split", "auuc", "qini")):
+        return "experimental_protocol"
+    if any(token in lowered for token in ("contribution", "we propose", "our method", "framework", "model")):
+        return "contribution_or_method_claim"
+    if any(token in lowered for token in ("existing", "requires", "problem", "gap", "lack", "overfit")):
+        return "problem_or_gap"
+    if section_type == "method":
+        return "method_exposition"
+    if section_type == "experiments":
+        return "experiment_exposition"
+    return "section_exposition"
+
+
+def split_paragraphs(text: str) -> list[str]:
+    chunks = re.split(r"\n\s*\n", text)
+    paragraphs: list[str] = []
+    for chunk in chunks:
+        cleaned = "\n".join(line.strip() for line in chunk.splitlines() if line.strip())
+        if cleaned:
+            paragraphs.append(cleaned)
+    return paragraphs
+
+
+def caption_match(line: str) -> re.Match[str] | None:
+    return re.match(
+        r"^(?P<kind>Table|TABLE|Figure|FIGURE|Fig\.?)\s+(?P<id>[A-Za-z0-9.]+)\s*[:.\-]?\s*(?P<caption>.*)$",
+        line.strip(),
+    )
+
+
+def classify_asset_role(asset_type: str, caption: str) -> str:
+    lowered = caption.lower()
+    if "ablation" in lowered or "component" in lowered:
+        return "ablation"
+    if any(token in lowered for token in ("coverage", "interval", "trade-off", "tradeoff", "sensitivity", "shift")):
+        return "uncertainty_or_tradeoff"
+    if any(token in lowered for token in ("main", "result", "performance", "ranking", "auuc", "qini")):
+        return "main_results"
+    if any(token in lowered for token in ("dataset", "statistics", "summary", "treatment assignment")):
+        return "dataset_or_protocol"
+    if any(token in lowered for token in ("overview", "framework", "pipeline", "architecture", "model")):
+        return "method_overview"
+    if any(token in lowered for token in ("case", "example")):
+        return "case_analysis"
+    return "supporting_evidence" if asset_type == "table" else "conceptual_visual"
+
+
+def caption_needs_continuation(caption: str) -> bool:
+    caption = caption.strip()
+    return bool(caption) and len(caption) < 140 and not re.search(r"[.!?)]$", caption)
+
+
+def extend_caption(caption: str, lines: list[str], start_index: int) -> str:
+    parts = [caption.strip()]
+    next_index = start_index + 1
+    while caption_needs_continuation(" ".join(parts)) and next_index < len(lines) and len(parts) < 4:
+        candidate = lines[next_index].strip()
+        next_index += 1
+        if not candidate:
+            continue
+        if candidate.startswith("[Page ") or is_caption_line(candidate) or is_heading_line(candidate):
+            break
+        if len(candidate) > 180:
+            break
+        parts.append(candidate)
+    return re.sub(r"\s+", " ", " ".join(parts)).strip()
+
+
+def asset_role_text(evidence_role: str) -> tuple[str, str, str, str]:
+    mapping = {
+        "main_results": (
+            "answers the primary comparison question",
+            "state the comparison question and ranking metric before the table",
+            "interpret ranking, margin, and evidence strength after the table",
+            "place after protocol, metrics, and comparator definitions",
+        ),
+        "ablation": (
+            "isolates which component supports the claimed gain",
+            "name the design choice being tested before the asset",
+            "explain the component-level implication after the asset",
+            "place after main results so the reader first sees the headline effect",
+        ),
+        "dataset_or_protocol": (
+            "defines the evaluation substrate and comparison scope",
+            "tell the reader why the dataset or protocol matters",
+            "connect the protocol to the later comparison table",
+            "place before baseline and result interpretation",
+        ),
+        "method_overview": (
+            "orients the reader to the method pipeline",
+            "introduce the mechanism or module sequence before the figure",
+            "map visual elements to later method prose after the figure",
+            "place near the beginning of the method section",
+        ),
+        "uncertainty_or_tradeoff": (
+            "explains a robustness, uncertainty, or trade-off pattern",
+            "state the trade-off or diagnostic question before the asset",
+            "explain the mechanism or limitation revealed by the asset",
+            "place after the main result table or in analysis/sensitivity subsections",
+        ),
+    }
+    return mapping.get(
+        evidence_role,
+        (
+            "supports a local subsection claim",
+            "state the local question before the asset",
+            "explain the takeaway after the asset",
+            "place next to the subsection claim it supports",
+        ),
+    )
+
+
+def extract_assets(sections: list[SectionRecord]) -> list[AssetRecord]:
+    assets: list[AssetRecord] = []
+    for section in sections:
+        lines = section.content.splitlines()
+        for index, line in enumerate(lines):
+            match = caption_match(line)
+            if not match:
+                continue
+            raw_kind = match.group("kind").lower()
+            asset_type = "table" if raw_kind.startswith("table") else "figure"
+            caption = match.group("caption").strip() or line.strip()
+            caption = extend_caption(caption, lines, index)
+            evidence_role = classify_asset_role(asset_type, caption)
+            local_role, before, after, placement = asset_role_text(evidence_role)
+            assets.append(
+                AssetRecord(
+                    asset_type=asset_type,
+                    asset_id=match.group("id"),
+                    caption=short_excerpt(caption, 180),
+                    appears_in_section=section.section_type,
+                    appears_in_title=section.title,
+                    evidence_role=evidence_role,
+                    local_role=local_role,
+                    placement_logic=placement,
+                    text_bridge_before=before,
+                    text_bridge_after=after,
+                    reuse_guidance="Reuse the asset function and placement logic only; do not copy caption wording or claims.",
+                )
+            )
+    return assets
+
+
+def build_paragraph_roles(sections: list[SectionRecord]) -> list[dict]:
+    roles: list[dict] = []
+    for section in sections:
+        for index, paragraph in enumerate(split_paragraphs(section.content), start=1):
+            roles.append(
+                {
+                    "section": section.section_type,
+                    "section_title": section.title,
+                    "paragraph_index": index,
+                    "role": paragraph_role(section.section_type, paragraph),
+                    "excerpt": short_excerpt(paragraph),
+                    "reuse_guidance": "Reuse the paragraph function, not the source wording.",
+                }
+            )
+    return roles
+
+
+def section_slots(section: SectionRecord, paragraph_roles: list[dict], assets: list[AssetRecord]) -> list[str]:
+    slots: list[str] = []
+    title = re.sub(r"^\d+(?:\.\d+)*\.?\s+", "", section.title).strip()
+    if title and title.lower() not in {"experiments", "experiment", "results"}:
+        slots.append(title.lower())
+    for role in paragraph_roles:
+        if role["section_title"] == section.title and role["role"] not in slots:
+            slots.append(role["role"].replace("_", " "))
+    for asset in assets:
+        if asset.appears_in_title == section.title:
+            slots.append(f"{asset.asset_type}: {asset.evidence_role}")
+    return slots
+
+
+def write_paper_artifacts(
+    paper_dir: Path,
+    source: str,
+    materialized_path: Path,
+    source_kind: str,
+    title: str,
+    method: str,
+    page_count: int | None,
+    sections: list[SectionRecord],
+    roles: list[dict],
+    assets: list[AssetRecord],
+) -> dict:
+    paper_dir.mkdir(parents=True, exist_ok=True)
+    metadata = {
+        "source": source,
+        "source_kind": source_kind,
+        "materialized_path": str(materialized_path),
+        "title": title,
+        "extraction_method": method,
+        "page_count": page_count,
+        "section_count": len(sections),
+        "visual_asset_count": len(assets),
+    }
+    (paper_dir / "metadata.json").write_text(json.dumps(metadata, indent=2, ensure_ascii=False), encoding="utf-8")
+    (paper_dir / "section-map.json").write_text(
+        json.dumps(
+            [
+                {
+                    key: value
+                    for key, value in asdict(section).items()
+                    if key != "content"
+                }
+                for section in sections
+            ],
+            indent=2,
+            ensure_ascii=False,
+        ),
+        encoding="utf-8",
+    )
+    (paper_dir / "paragraph-roles.json").write_text(json.dumps(roles, indent=2, ensure_ascii=False), encoding="utf-8")
+    (paper_dir / "visual-assets.json").write_text(
+        json.dumps([asdict(asset) for asset in assets], indent=2, ensure_ascii=False),
+        encoding="utf-8",
+    )
+    write_section_logic(paper_dir / "section-logic.md", title, sections, roles, assets)
+    write_writing_patterns(paper_dir / "writing-patterns.md", title, sections, roles, assets)
+    (paper_dir / "extraction-report.md").write_text(
+        "\n".join(
+            [
+                f"# Extraction Report: {title}",
+                "",
+                f"- Source: `{source}`",
+                f"- Extraction method: `{method}`",
+                f"- Sections detected: {len(sections)}",
+                f"- Visual/table assets detected: {len(assets)}",
+                "- Boundary: this artifact extracts reusable structure only; it must not be used as evidence for the current paper.",
+            ]
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+    return metadata
+
+
+def write_section_logic(path: Path, title: str, sections: list[SectionRecord], roles: list[dict], assets: list[AssetRecord]) -> None:
+    lines = [f"# Section Logic: {title}", ""]
+    for section in sections:
+        section_roles = [role for role in roles if role["section_title"] == section.title]
+        section_assets = [asset for asset in assets if asset.appears_in_title == section.title]
+        lines.append(f"## {section.title}")
+        lines.append("")
+        lines.append(f"- Section type: `{section.section_type}`")
+        if section_roles:
+            role_list = ", ".join(dict.fromkeys(role["role"] for role in section_roles))
+            lines.append(f"- Paragraph roles: {role_list}")
+        if section_assets:
+            asset_list = ", ".join(f"{asset.asset_type}:{asset.evidence_role}" for asset in section_assets)
+            lines.append(f"- Asset roles: {asset_list}")
+        lines.append("- Reuse: preserve the slot order and rhetorical function, not the wording.")
+        lines.append("")
+    path.write_text("\n".join(lines).strip() + "\n", encoding="utf-8")
+
+
+def write_writing_patterns(path: Path, title: str, sections: list[SectionRecord], roles: list[dict], assets: list[AssetRecord]) -> None:
+    lines = [f"# Writing Patterns: {title}", ""]
+    lines.append("## Template Use")
+    lines.append("")
+    lines.append("- Use this paper as one structural template among several references.")
+    lines.append("- Reproduce section slots, paragraph functions, and asset placement logic only.")
+    lines.append("- Do not copy wording, claims, metrics, or conclusions.")
+    lines.append("")
+    lines.append("## Observed Section Slots")
+    lines.append("")
+    for section in sections:
+        lines.append(f"- {section.section_type}: {section.title}")
+    lines.append("")
+    lines.append("## Visual/Table Pattern")
+    lines.append("")
+    if assets:
+        for asset in assets:
+            lines.append(f"- {asset.asset_type} {asset.asset_id}: {asset.evidence_role} -> {asset.local_role}")
+    else:
+        lines.append("- No table or figure caption was detected.")
+    path.write_text("\n".join(lines).strip() + "\n", encoding="utf-8")
+
+
+def merge_unique(values: list[str]) -> list[str]:
+    seen: set[str] = set()
+    merged: list[str] = []
+    for value in values:
+        normalized = re.sub(r"\s+", " ", value.strip())
+        key = normalized.lower()
+        if not normalized or key in seen:
+            continue
+        seen.add(key)
+        merged.append(normalized)
+    return merged
+
+
+def normalized_section_title(title: str) -> str:
+    title = re.sub(r"^\d+(?:\.\d+)*\.?\s+", "", title.strip())
+    title = re.sub(r"^[A-Z]\.\d+(?:\.\d+)*\.?\s+", "", title)
+    title = re.sub(r"^Appendix\s+[A-Z](?:\.\d+)*\.?\s*[:\-]?\s*", "", title, flags=re.IGNORECASE)
+    return re.sub(r"\s+", " ", title).strip().lower()
+
+
+def experiment_slots_from_signal(section: SectionRecord, role: str, text: str) -> list[str]:
+    title = normalized_section_title(section.title)
+    lowered = f"{title} {text.lower()}"
+    is_appendix = section.section_type == "appendix"
+    slots: list[str] = []
+
+    if is_appendix and "dataset" in lowered and any(
+        token in lowered for token in ("statistics", "statistic", "summary", "sample", "feature")
+    ):
+        slots.append("appendix_dataset_statistics")
+    if is_appendix and any(token in lowered for token in ("baseline", "metric", "implementation", "hyperparameter")):
+        slots.append("appendix_baseline_metric_details")
+    if role == "dataset_statistics" or (
+        "dataset" in lowered and any(token in lowered for token in ("statistics", "summary", "sample", "feature"))
+    ):
+        slots.append("dataset_statistics")
+    if role == "dataset_description" or "datasets" in title or "dataset" in title:
+        slots.append("dataset_description")
+    if role == "split_protocol" or any(
+        token in lowered
+        for token in ("split", "seed", "sampling", "train/test", "validation", "protocol")
+    ):
+        slots.append("split_protocol")
+    if role == "baseline_setup" or "baseline" in lowered or "comparator" in lowered:
+        slots.append("baseline_setup")
+    if role == "metric_definition" or "metric" in lowered or "auuc" in lowered or "qini" in lowered:
+        slots.append("metric_definition")
+    if role == "implementation_details" or any(
+        token in lowered
+        for token in ("implementation", "hyperparameter", "tuning", "epoch", "learning rate", "hardware")
+    ):
+        slots.append("implementation_details")
+    if "ablation" in lowered:
+        slots.append("ablation")
+    if "sensitivity" in lowered or "shift" in lowered or "trade-off" in lowered or "tradeoff" in lowered:
+        slots.append("sensitivity")
+    if role == "result_interpretation" or any(
+        token in lowered for token in ("main result", "overall performance", "performance", "results and discussion")
+    ):
+        slots.append("main_results")
+    return list(dict.fromkeys(slots))
+
+
+def is_experiment_protocol_section(section: SectionRecord) -> bool:
+    if section.section_type in {"experiments", "discussion"}:
+        return True
+    if section.section_type != "appendix":
+        return False
+    title = normalized_section_title(section.title)
+    return any(
+        token in title
+        for token in (
+            "dataset",
+            "baseline",
+            "metric",
+            "experiment",
+            "experimental",
+            "setup",
+            "result",
+            "ablation",
+            "sensitivity",
+            "complexity",
+            "online",
+        )
+    )
+
+
+def slot_payload(
+    *,
+    source_paper: str,
+    slot: str,
+    section: SectionRecord,
+    evidence_excerpt: str,
+    paragraph_index: int | None = None,
+    asset_type: str | None = None,
+    asset_id: str | None = None,
+) -> dict:
+    guidance = EXPERIMENT_PROTOCOL_SLOT_GUIDANCE[slot]
+    payload = {
+        "slot": slot,
+        "source_paper": source_paper,
+        "source_heading": section.title,
+        "source_section_type": section.section_type,
+        "paragraph_index": paragraph_index,
+        "evidence_excerpt": short_excerpt(evidence_excerpt),
+        "reader_question": guidance["reader_question"],
+        "placement_guidance": guidance["placement_guidance"],
+        "linked_main_section": "experiments" if slot.startswith("appendix_") else "",
+        "reuse_guidance": "Reuse this protocol role and placement logic only; do not copy wording, claims, metrics, data, or conclusions.",
+    }
+    if asset_type and asset_id:
+        payload["asset_type"] = asset_type
+        payload["asset_id"] = asset_id
+    return payload
+
+
+def build_experiment_protocol_slots_for_payload(payload: dict) -> list[dict]:
+    slots: list[dict] = []
+    sections_by_title = {section.title: section for section in payload["sections"]}
+
+    for role in payload["roles"]:
+        section = sections_by_title.get(role["section_title"])
+        if not section:
+            continue
+        if not is_experiment_protocol_section(section):
+            continue
+        role_slots = experiment_slots_from_signal(section, role["role"], role["excerpt"])
+        if not role_slots:
+            continue
+        for slot in role_slots:
+            slots.append(
+                slot_payload(
+                    source_paper=payload["slug"],
+                    slot=slot,
+                    section=section,
+                    paragraph_index=role["paragraph_index"],
+                    evidence_excerpt=role["excerpt"],
+                )
+            )
+
+    for asset in payload["assets"]:
+        section = sections_by_title.get(asset.appears_in_title)
+        if not section:
+            continue
+        if not is_experiment_protocol_section(section):
+            continue
+        asset_slots = experiment_slots_from_signal(section, asset.evidence_role, asset.caption)
+        if asset.evidence_role == "dataset_or_protocol" and section.section_type == "appendix":
+            asset_slots = ["appendix_dataset_statistics"]
+        if not asset_slots:
+            continue
+        for slot in asset_slots:
+            slots.append(
+                slot_payload(
+                    source_paper=payload["slug"],
+                    slot=slot,
+                    section=section,
+                    evidence_excerpt=asset.caption,
+                    asset_type=asset.asset_type,
+                    asset_id=asset.asset_id,
+                )
+            )
+
+    seen: set[tuple[str, str, str, str]] = set()
+    unique_slots: list[dict] = []
+    for item in slots:
+        key = (
+            item["source_paper"],
+            item["slot"],
+            item["source_heading"].lower(),
+            item["evidence_excerpt"].lower(),
+        )
+        if key in seen:
+            continue
+        seen.add(key)
+        unique_slots.append(item)
+    return unique_slots
+
+
+def build_section_templates(output_dir: Path, paper_payloads: list[dict]) -> None:
+    target = output_dir / "section-templates"
+    target.mkdir(parents=True, exist_ok=True)
+    section_types = ["abstract", "introduction", "related-work", "method", "experiments", "discussion", "conclusion"]
+    for section_type in section_types:
+        source_papers: list[str] = []
+        observed_titles: list[str] = []
+        slots: list[str] = []
+        paragraph_roles: list[str] = []
+        asset_roles: list[dict] = []
+        for payload in paper_payloads:
+            matching_sections = [section for section in payload["sections"] if section.section_type == section_type]
+            if section_type == "experiments":
+                matching_sections.extend(
+                    section for section in payload["sections"] if section.section_type in {"discussion"}
+                )
+            if not matching_sections:
+                continue
+            source_papers.append(payload["slug"])
+            for section in matching_sections:
+                observed_titles.append(section.title)
+                slots.extend(section_slots(section, payload["roles"], payload["assets"]))
+            paragraph_roles.extend(
+                role["role"] for role in payload["roles"] if role["section"] == section_type
+            )
+            asset_roles.extend(
+                {
+                    "asset_type": asset.asset_type,
+                    "evidence_role": asset.evidence_role,
+                    "local_role": asset.local_role,
+                    "source_paper": payload["slug"],
+                }
+                for asset in payload["assets"]
+                if asset.appears_in_section == section_type
+                or (section_type == "experiments" and asset.appears_in_section in {"experiments", "discussion"})
+            )
+        if not source_papers:
+            continue
+        template = {
+            "section": section_type,
+            "template_id": f"{section_type}-multi-reference-template",
+            "source_papers": merge_unique(source_papers),
+            "observed_titles": merge_unique(observed_titles),
+            "section_slots": merge_unique(slots),
+            "paragraph_roles": merge_unique(paragraph_roles),
+            "asset_roles": asset_roles,
+            "reuse_rule": "Reuse structure only; do not copy wording, claims, metrics, or conclusions from reference papers.",
+        }
+        if section_type == "experiments":
+            protocol_slots: list[dict] = []
+            for payload in paper_payloads:
+                protocol_slots.extend(build_experiment_protocol_slots_for_payload(payload))
+            template["experiment_protocol_slots"] = protocol_slots
+        (target / f"{section_type}.json").write_text(
+            json.dumps(template, indent=2, ensure_ascii=False),
+            encoding="utf-8",
+        )
+        if section_type == "experiments":
+            (target / "experiments-protocol.json").write_text(
+                json.dumps(
+                    {
+                        "section": "experiments",
+                        "template_id": "experiments-protocol-slots",
+                        "source_papers": template["source_papers"],
+                        "experiment_protocol_slots": template["experiment_protocol_slots"],
+                        "reuse_rule": "Reuse experiment setup topology only: dataset, split, baseline, metric, implementation, result, ablation, sensitivity, and appendix-link roles.",
+                    },
+                    indent=2,
+                    ensure_ascii=False,
+                ),
+                encoding="utf-8",
+            )
+
+
+def build_visual_templates(output_dir: Path, paper_payloads: list[dict]) -> None:
+    target = output_dir / "visual-templates"
+    target.mkdir(parents=True, exist_ok=True)
+    asset_roles: list[dict] = []
+    for payload in paper_payloads:
+        for asset in payload["assets"]:
+            if asset.appears_in_section in {"experiments", "discussion", "method"}:
+                item = asdict(asset)
+                item["source_paper"] = payload["slug"]
+                asset_roles.append(item)
+    template = {
+        "template_id": "experiment-visual-and-table-template",
+        "asset_roles": asset_roles,
+        "reuse_rule": "Use table and figure functions, placement, and bridge logic to plan current paper assets; do not copy captions or data.",
+    }
+    (target / "experiment-assets.json").write_text(
+        json.dumps(template, indent=2, ensure_ascii=False),
+        encoding="utf-8",
+    )
+
+
+def write_aggregate_playbook(output_dir: Path, paper_payloads: list[dict]) -> None:
+    lines = [
+        "# Aggregate Template Playbook",
+        "",
+        "Purpose: help `/lab:write` reproduce mature paper structure from multiple reference templates.",
+        "",
+        "## Sources",
+        "",
+    ]
+    for payload in paper_payloads:
+        lines.append(f"- `{payload['slug']}`: {payload['title']}")
+    lines.extend(
+        [
+            "",
+            "## Reuse Boundary",
+            "",
+            "- Reproduce section order, paragraph roles, table/figure function, and bridge logic.",
+            "- Do not copy wording, claims, experimental conclusions, metrics, or terminology names.",
+            "- If only one reference supports a structure, treat it as a single-template pattern, not a universal rule.",
+            "",
+            "## Multi-Template Write Procedure",
+            "",
+            "1. Pick 2-3 closest section templates for the current paper section.",
+            "2. For experiment sections, preserve protocol slots when present: datasets, splits, baselines, metrics, implementation details, main results, ablations, sensitivity analysis, and appendix links.",
+            "3. Build a mini-outline from common slots and current-paper evidence.",
+            "4. Add required table/figure assets with local before/after bridge functions.",
+            "5. Draft with current-paper terminology and evidence only.",
+            "",
+            "## Table/Figure Planning Rule",
+            "",
+            "Every major table or figure should answer a reader question, appear near the subsection claim it supports, and have one bridge sentence before and one interpretation sentence after it.",
+        ]
+    )
+    (output_dir / "aggregate-template-playbook.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def process_source(source: str, output_dir: Path, cache_dir: Path) -> dict:
+    materialized_path, source_kind = materialize_source(source, cache_dir)
+    if not materialized_path.exists():
+        raise FileNotFoundError(f"source not found: {source}")
+    text, method, page_count = read_source_text(materialized_path)
+    title = detect_title(text, materialized_path.stem)
+    slug = slugify(title or materialized_path.stem)
+    sections = split_sections(text)
+    roles = build_paragraph_roles(sections)
+    assets = extract_assets(sections)
+    metadata = write_paper_artifacts(
+        output_dir / slug,
+        source,
+        materialized_path,
+        source_kind,
+        title,
+        method,
+        page_count,
+        sections,
+        roles,
+        assets,
+    )
+    return {
+        "slug": slug,
+        "title": title,
+        "metadata": metadata,
+        "sections": sections,
+        "roles": roles,
+        "assets": assets,
+    }
+
+
+def main() -> int:
+    args = parse_args()
+    output_dir = Path(args.output_dir).expanduser().resolve()
+    output_dir.mkdir(parents=True, exist_ok=True)
+    cache_dir = output_dir / ".cache"
+
+    payloads: list[dict] = []
+    source_entries: list[dict] = []
+    try:
+        for source in args.sources:
+            payload = process_source(source, output_dir, cache_dir)
+            payloads.append(payload)
+            source_entries.append(
+                {
+                    "source": source,
+                    "slug": payload["slug"],
+                    "title": payload["title"],
+                    "artifact_dir": str(output_dir / payload["slug"]),
+                }
+            )
+    except RuntimeError as exc:
+        if "No PDF text extractor succeeded" in str(exc):
+            bootstrapped = rerun_with_uv_pypdf()
+            if bootstrapped is not None:
+                return bootstrapped
+        raise
+
+    (output_dir / "sources.json").write_text(
+        json.dumps({"sources": source_entries}, indent=2, ensure_ascii=False),
+        encoding="utf-8",
+    )
+    build_section_templates(output_dir, payloads)
+    build_visual_templates(output_dir, payloads)
+    write_aggregate_playbook(output_dir, payloads)
+    print(f"reference template patterns written to {output_dir}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())