lede 0.3.0__py3-none-any.whl

lede/__init__.py

@@ -0,0 +1,28 @@
+"""lede — deterministic extractive summarization.
+
+Public API:
+  summarize(text, max_length, mode='default') -> SummaryResult
+  brief(text, *, overview_max, max_facts, include_phrases, format) -> str | dict
+  clean_text(text) -> str
+  strip_think(text) -> str
+  extract_keyword(text, keywords, num_sentences=10) -> str
+  set_default_backend(name) -> None   # enrichment backend selector
+"""
+from lede.clean import clean_text, strip_think
+from lede.tfidf import summarize
+from lede.brief import brief
+from lede.keyword import extract_keyword
+from lede._types import SummaryResult
+from lede.extract._backends import set_default_backend
+
+__version__ = "0.3.0"
+__all__ = [
+    "summarize",
+    "brief",
+    "clean_text",
+    "strip_think",
+    "extract_keyword",
+    "SummaryResult",
+    "set_default_backend",
+    "__version__",
+]

lede/_headings.py

@@ -0,0 +1,111 @@
+"""Shared heading detection.
+
+Any sentence matching any of these patterns is considered a heading and is
+dropped from candidate selection in mode='default'. Also used by
+extract.outline to discover section names.
+"""
+import re
+
+# Markdown ATX-style heading: one or more # followed by space and text
+_MD_HEADING_RE = re.compile(r"^\s*#+\s+.+$")
+
+# ALL-CAPS short line: 4-80 chars of A-Z/space/colon, no lowercase.
+# Upper bound extended from 28 -> 80 in T13b so long document-title
+# conventions like "SUPREME COURT OF THE UNITED STATES" (32 chars) match.
+_ALLCAPS_RE = re.compile(r"^\s*[A-Z][A-Z\s]{3,80}:?\s*$")
+
+# Short label ending in colon (<=30 chars including the colon)
+_SHORT_LABEL_RE = re.compile(r"^\s*.{1,30}:\s*$")
+
+# Bare title-case single-line heading (T13b Class A): starts uppercase,
+# second char alphanumeric (not space/punct), up to ~60 chars total, no
+# internal special chars, no terminal punctuation. Matches bare lines like
+# "Abstract", "Introduction", "Modern methods", "Privacy Policy".
+#
+# The "no terminal punctuation" constraint is load-bearing: it excludes short
+# body sentences like "Costs declined." that ended in a period, which was the
+# T6 false-positive class we dropped the < 4 content-tokens fallback for.
+_BARE_TITLE_RE = re.compile(r"^\s*[A-Z][A-Za-z0-9][A-Za-z0-9 ]{0,58}$")
+
+# Numbered-section-prefix heading (T13b Class B): "\d+. Section Name" on its
+# own line. Matches "1. Information We Collect" etc. The numeric prefix is
+# stripped by `heading_name()` so the emitted name matches gold.
+_NUMBERED_SECTION_RE = re.compile(r"^\s*\d+\.\s+[A-Z][A-Za-z0-9 ]{0,58}$")
+
+# Title-with-dash heading (T13d): document title line of the form
+# "Title — Metadata" (em-dash U+2014, en-dash U+2013, or hyphen-minus), e.g.
+# "Privacy Policy — Effective Date: 2026-01-01". Authorized by the labeling
+# protocol's Edge-case conventions (docs/extraction-gold-labeling.md).
+# Requires a space on both sides of the dash so hyphenated single words
+# ("state-of-the-art") don't match.
+#
+# The trailing character class `[^.!?]*$` (no terminal `.`/`!`/`?`) mirrors
+# the `_BARE_TITLE_RE` safety constraint — excludes body sentences like
+# "Main concern is pricing — our $50K quote is 40% above their Q2 budget."
+# that would otherwise match. The dash-metadata suffix is stripped by
+# `heading_name()`.
+_TITLE_WITH_DASH_RE = re.compile(r"^\s*[A-Z][A-Za-z0-9 ]{0,58}\s+[—–\-]\s+\S[^.!?]*$")
+
+
+# Markdown heading depth helper — stripped from `## Section` etc.
+_MD_DEPTH_RE = re.compile(r"^\s*(#+)\s+")
+
+
+def is_structural_heading(sentence: str) -> bool:
+    """True when `sentence` matches a structural heading pattern.
+
+    Narrower than [`is_heading`] — drops the "< 4 content tokens"
+    fallback so short body sentences don't masquerade as headings.
+    Use this when you only want explicit document structure (markdown,
+    title-case, all-caps, etc.) and not the heuristic short-token rule.
+    """
+    if not sentence.strip():
+        return False
+    if _MD_HEADING_RE.match(sentence):
+        return True
+    if _ALLCAPS_RE.match(sentence):
+        return True
+    if _SHORT_LABEL_RE.match(sentence):
+        return True
+    if _BARE_TITLE_RE.match(sentence):
+        return True
+    if _NUMBERED_SECTION_RE.match(sentence):
+        return True
+    if _TITLE_WITH_DASH_RE.match(sentence):
+        return True
+    return False
+
+
+def md_depth(heading_line: str) -> int:
+    """Markdown heading depth (# = 1, ## = 2, ...). 1 for non-markdown headings."""
+    m = _MD_DEPTH_RE.match(heading_line)
+    if m:
+        return len(m.group(1))
+    return 1
+
+
+def is_heading(sentence: str) -> bool:
+    """True when `sentence` matches any heading pattern (incl. heuristic)."""
+    if is_structural_heading(sentence):
+        return True
+    if not sentence.strip():
+        return False
+    # Fewer than 4 content-word tokens (rough "title-like" filter).
+    toks = re.findall(r"[A-Za-z]{3,}", sentence)
+    return len(toks) < 4
+
+
+def heading_name(sentence: str) -> str | None:
+    """Extract the name portion of a heading, or None if not a heading."""
+    s = sentence.strip()
+    if not s:
+        return None
+    # Strip markdown #'s, numeric-section prefix, em-dash metadata suffix,
+    # and trailing colon.
+    s = re.sub(r"^#+\s+", "", s)
+    s = re.sub(r"^\d+\.\s+", "", s)
+    s = re.sub(r"\s+[—–\-]\s+.*$", "", s)  # T13d: strip em-dash metadata
+    s = s.rstrip(":").strip()
+    if not s:
+        return None
+    return s

lede/_parity.py

@@ -0,0 +1,73 @@
+"""Canonical text formatters for v0.2 extract primitives — used by the
+cross-runtime parity walker (`rust/tests/fixtures.rs`) and the Python
+generator (`benchmarks/gen_parity_fixtures.py`).
+
+Each formatter takes the primitive's structured output and emits a
+deterministic, line-oriented text representation. Python and Rust
+have byte-identical implementations so a fixture generated from
+Python output can be byte-compared against the Rust output for the
+same input.
+
+Format choices:
+- One record per line.
+- Fields within a record separated by ` | ` (space-pipe-space).
+- Trailing newline at end of file (mirror of how Python's `\\n`.join
+  + final write_text behaves).
+- Empty-output → empty file (no trailing newline either).
+
+Mirrors: rust/src/_parity.rs.
+"""
+from __future__ import annotations
+
+from ._types import Metadata, PhraseFact, Section, Stat
+
+
+def _join_lines(lines: list[str]) -> str:
+    if not lines:
+        return ""
+    return "\n".join(lines) + "\n"
+
+
+def format_stats(stats: tuple[Stat, ...]) -> str:
+    lines = [
+        f"{s.stat_type} | {s.value} | {s.unit} | {s.phrase} | {s.context_sentence}"
+        for s in stats
+    ]
+    return _join_lines(lines)
+
+
+def format_outline(outline: tuple[Section, ...]) -> str:
+    lines = [
+        f"{sec.depth} | {sec.name} | {sec.representative_sentence}"
+        for sec in outline
+    ]
+    return _join_lines(lines)
+
+
+def format_toc(toc: tuple[str, ...]) -> str:
+    return _join_lines(list(toc))
+
+
+def format_metadata(md: Metadata) -> str:
+    return _join_lines([
+        "DATES: " + ", ".join(md.dates),
+        "AMOUNTS: " + ", ".join(md.amounts),
+        "URLS: " + ", ".join(md.urls),
+        "ENTITIES: " + ", ".join(md.entities),
+    ])
+
+
+def format_phrases(phrases: tuple[str, ...]) -> str:
+    return _join_lines(list(phrases))
+
+
+def format_key_facts(facts: tuple[str, ...]) -> str:
+    return _join_lines(list(facts))
+
+
+def format_correlate_facts(facts: tuple[PhraseFact, ...]) -> str:
+    lines = [
+        f"{f.entity} | {f.number} | {f.polarity} | {f.sentence}"
+        for f in facts
+    ]
+    return _join_lines(lines)

lede/_types.py

@@ -0,0 +1,47 @@
+"""v0.2.0 return types (frozen dataclasses)."""
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class Stat:
+    value: str
+    unit: str
+    phrase: str
+    context_sentence: str
+    stat_type: str  # "money" | "percent" | "count" | "date" | "duration"
+
+
+@dataclass(frozen=True)
+class Section:
+    depth: int
+    name: str
+    representative_sentence: str
+
+
+@dataclass(frozen=True)
+class Metadata:
+    dates: tuple[str, ...] = ()
+    amounts: tuple[str, ...] = ()
+    urls: tuple[str, ...] = ()
+    entities: tuple[str, ...] = ()  # populated only when lede[ner] installed (Task 9)
+
+
+@dataclass(frozen=True)
+class PhraseFact:
+    entity: str
+    number: str
+    polarity: str  # "absolute" | "growth" | "decline" | "unknown"
+    sentence: str
+
+
+@dataclass(frozen=True)
+class SummaryResult:
+    summary: str
+    stats: tuple[Stat, ...] | None = None
+    outline: tuple[Section, ...] | None = None
+    metadata: Metadata | None = None
+    phrases: tuple[str, ...] | None = None
+    correlated_facts: tuple[PhraseFact, ...] | None = None
+
+    def __str__(self) -> str:
+        return self.summary

lede/brief.py

@@ -0,0 +1,157 @@
+"""At-a-glance document brief — composes summarize + key_facts + toc.
+
+Top-level convenience primitive for "reader brief" use cases: email
+digests, file-browser previews, ingest-pipeline pre-summarization.
+Agnostic of document type — no per-doc heuristics. Callers who want
+different composition should call the underlying primitives directly.
+
+Mirrors rust/src/brief.rs. Keep the three output formats (string,
+markdown, dict) byte-identical across Python and Rust for the regex
+backend.
+"""
+from __future__ import annotations
+
+from .tfidf import summarize
+from .extract.key_facts import key_facts
+from .extract.outline import toc
+from .extract.phrases import phrases
+
+# Auto-detect the wordforms extra at import time. When available,
+# brief() forwards convert_word_names=True to its internal key_facts()
+# call so spelled-out numbers ("five thousand documents", "twelve lines")
+# surface in the key facts section. Mirrors the same pattern used in
+# benchmarks/extraction_eval.py::_STATS_WORDFORMS.
+try:
+    import text_to_num  # noqa: F401
+
+    _HAS_WORDFORMS = True
+except ImportError:
+    _HAS_WORDFORMS = False
+
+
+# Overview budget clamps. Floor keeps very short inputs readable; ceiling
+# keeps very long inputs actually brief (a 30 KB doc at 0.35 would produce
+# a 10 KB "overview" which defeats the purpose).
+_OVERVIEW_MIN_FRAC = 0.05
+_OVERVIEW_MAX_FRAC = 0.50
+_OVERVIEW_MIN_CHARS = 250
+_OVERVIEW_MAX_CHARS = 1500
+
+
+def brief(
+    text: str,
+    *,
+    overview_max: float = 0.35,
+    max_facts: int = 10,
+    include_phrases: bool = False,
+    convert_word_names: bool | None = None,
+    format: str = "string",
+) -> str | dict:
+    """Produce an at-a-glance brief of a document.
+
+    Composes ``summarize()`` (overview) + ``extract.key_facts()`` +
+    ``extract.toc()`` into a single caller-friendly artifact. Optional
+    ``extract.phrases()`` via ``include_phrases=True``.
+
+    Args:
+        text: input document text.
+        overview_max: fraction of source length to budget for the overview.
+            Clamped to ``[0.05, 0.50]``. Default 0.35. The resolved
+            char budget is further clamped to ``[250, 1500]`` so short
+            docs still get a readable overview and long docs stay brief.
+        max_facts: cap on key-facts sentences. Default 10.
+        include_phrases: when True, append a key-phrases section (regex
+            backend). Default False.
+        convert_word_names: forward to ``key_facts()`` so spelled-out
+            numbers ("five thousand documents") surface in the
+            key-facts section. ``None`` (default) auto-detects whether
+            ``text2num`` is importable. Pass ``True`` / ``False``
+            explicitly to lock the behavior — useful when you need
+            output to match a Rust binary built with or without the
+            ``wordforms`` cargo feature regardless of which Python
+            extras happen to be installed.
+        format: output shape. One of:
+            - ``"string"`` (default) — plain text with section labels.
+            - ``"markdown"`` — ``##`` headers + bullet lists.
+            - ``"dict"`` — structured dict with overview/key_facts/toc/phrases.
+
+    Returns:
+        ``str`` for ``"string"`` / ``"markdown"`` formats, ``dict`` for
+        ``"dict"`` format.
+
+    Raises:
+        ValueError: when ``format`` is not one of the three supported values.
+
+    Notes on cross-runtime parity:
+        Python's auto-detect (``convert_word_names=None``) is at import
+        time; Rust's equivalent is the compile-time ``wordforms`` cargo
+        feature. The two can disagree silently when (a) the Python
+        process has ``text2num`` installed for an unrelated tool and
+        (b) the Rust binary was built without ``--features wordforms``
+        — same input, different brief bytes. Pass an explicit
+        ``convert_word_names=True`` (or ``False``) on both sides to
+        lock parity.
+    """
+    # Clamp overview_max to sane fraction bounds.
+    overview_max = max(_OVERVIEW_MIN_FRAC, min(_OVERVIEW_MAX_FRAC, overview_max))
+    budget = int(len(text) * overview_max)
+    budget = max(_OVERVIEW_MIN_CHARS, min(_OVERVIEW_MAX_CHARS, budget))
+
+    overview_result = summarize(text, max_length=budget)
+    overview_text = overview_result.summary.rstrip()
+
+    use_wordforms = _HAS_WORDFORMS if convert_word_names is None else convert_word_names
+    facts = key_facts(
+        text,
+        max_facts=max_facts,
+        convert_word_names=use_wordforms,
+    )
+
+    sections = toc(text)
+
+    phrases_list: tuple[str, ...] = phrases(text) if include_phrases else ()
+
+    if format == "dict":
+        return {
+            "overview": overview_text,
+            "key_facts": list(facts),
+            "toc": list(sections),
+            "phrases": list(phrases_list) if include_phrases else None,
+        }
+    if format == "markdown":
+        parts: list[str] = ["## Overview\n", overview_text, ""]
+        if facts:
+            parts.append("\n## Key facts\n")
+            for f in facts:
+                parts.append(f"- {f}")
+            parts.append("")
+        if sections:
+            parts.append("\n## Also in this doc\n")
+            for s in sections:
+                parts.append(f"- {s}")
+            parts.append("")
+        if include_phrases and phrases_list:
+            parts.append("\n## Key phrases\n")
+            parts.append("  ·  ".join(phrases_list))
+            parts.append("")
+        return "\n".join(parts)
+    if format == "string":
+        parts = ["Overview:", overview_text, ""]
+        if facts:
+            parts.append("Key facts:")
+            for f in facts:
+                parts.append(f"  - {f}")
+            parts.append("")
+        if sections:
+            parts.append("Also in this doc:")
+            for s in sections:
+                parts.append(f"  - {s}")
+            parts.append("")
+        if include_phrases and phrases_list:
+            parts.append("Key phrases:")
+            parts.append("  " + ", ".join(phrases_list))
+            parts.append("")
+        return "\n".join(parts).rstrip() + "\n"
+    raise ValueError(
+        f"format must be 'string', 'markdown', or 'dict'; got {format!r}"
+    )

lede/clean.py

@@ -0,0 +1,100 @@
+"""Text cleaners: clean_text (markdown + filler + boilerplate) and strip_think
+(reasoning-model <think>...</think> blocks).
+
+Both functions are deterministic and stdlib-only.
+"""
+import re
+
+_THINK_RE = re.compile(r"<think>.*?</think>\s*", re.DOTALL)
+
+
+def strip_think(text: str) -> str:
+    """Remove <think>...</think> blocks and trim surrounding whitespace."""
+    if text is None:
+        return ""
+    return _THINK_RE.sub("", text).strip()
+
+
+# --- clean_text — pipeline ---
+#
+# Steps in order:
+#   1. Strip markdown (*, _, #, ---, bullets, numbered list prefixes)
+#   2. Remove filler phrases
+#   3. Remove filler words
+#   4. Remove CRM boilerplate
+#   5. Lowercase
+#   6. Collapse whitespace and blank lines
+#   7. Trim
+
+_FILLER_PHRASES = re.compile(
+    r"\b(just wanted to|i just wanted to|wanted to follow up|as discussed"
+    r"|per our conversation|as mentioned|going forward|at the end of the day"
+    r"|in terms of|with respect to|in regards to|please find attached"
+    r"|hope this helps|let me know if you have any questions"
+    r"|looking forward to hearing from you)\b",
+    re.IGNORECASE,
+)
+
+_FILLER_WORDS = re.compile(
+    r"\b(basically|essentially|actually|literally|honestly|frankly"
+    r"|obviously|clearly|simply|really|very|quite|rather|pretty much"
+    r"|kind of|sort of|in order to|due to the fact that"
+    r"|at this point in time|for all intents and purposes)\b",
+    re.IGNORECASE,
+)
+
+_CRM_PATTERNS = [
+    re.compile(r"\bNo update[s]?\b\.?", re.IGNORECASE),
+    re.compile(r"\bCalendar invite sent[.]?\b", re.IGNORECASE),
+    re.compile(
+        r"\bSent (proposal|case study|documentation|overview|pricing)"
+        r" (documentation |via email|as requested)?\.?\b",
+        re.IGNORECASE,
+    ),
+    re.compile(r"\bWaiting (on|for) callback\.?\b", re.IGNORECASE),
+    re.compile(r"\bUpdated CRM with latest info\.?\b", re.IGNORECASE),
+    re.compile(r"\bMeeting confirmed for next week\.?\b", re.IGNORECASE),
+    re.compile(r"\bFollowing standard sales process\.?\b", re.IGNORECASE),
+    re.compile(r"\bMeeting went as expected\.?\b", re.IGNORECASE),
+]
+
+
+def clean_text(text: str | None) -> str:
+    """Strip markdown, filler phrases, filler words, and CRM boilerplate;
+    lowercase and normalize whitespace.
+
+    Returns the empty string for None or empty input.
+    """
+    if not text:
+        return ""
+
+    result = text
+
+    # 1. Markdown formatting
+    result = re.sub(r"\*{1,3}", "", result)
+    result = re.sub(r"_{1,3}", "", result)
+    result = re.sub(r"^#{1,6}\s*", "", result, flags=re.MULTILINE)
+    result = re.sub(r"^-{3,}$", "", result, flags=re.MULTILINE)
+    result = re.sub(r"^\s*[-*+]\s+", "", result, flags=re.MULTILINE)
+    result = re.sub(r"^\s*\d+\.\s+", "", result, flags=re.MULTILINE)
+
+    # 2-3. Filler
+    result = _FILLER_PHRASES.sub("", result)
+    result = _FILLER_WORDS.sub("", result)
+
+    # 4. CRM boilerplate
+    for pattern in _CRM_PATTERNS:
+        result = pattern.sub("", result)
+
+    # 5. Lowercase
+    result = result.lower()
+
+    # 6. Whitespace normalization
+    result = re.sub(r"[ \t]+", " ", result)
+    result = re.sub(r"\n\s*\n+", "\n", result)
+    result = re.sub(r"^\s+", "", result, flags=re.MULTILINE)
+    result = re.sub(r"\s+$", "", result, flags=re.MULTILINE)
+    result = re.sub(r"^\s*$\n?", "", result, flags=re.MULTILINE)
+
+    # 7. Trim
+    return result.strip()

lede/cli.py

@@ -0,0 +1,84 @@
+"""lede CLI.
+
+Usage:
+  lede [FILE] --mode {tfidf,keyword,clean_text,strip_think} [OPTIONS]
+
+Reads FILE or stdin, writes summary to stdout.
+"""
+import argparse
+import sys
+from pathlib import Path
+
+from lede import summarize, clean_text, strip_think, extract_keyword
+
+
+def _read_input(path: str | None) -> str:
+    # Force UTF-8 — relying on locale gives Windows users mojibake on
+    # non-ASCII content and breaks the byte-identical-runtime claim
+    # (the Rust CLI reads files as bytes and decodes UTF-8 explicitly).
+    if path and path != "-":
+        return Path(path).read_text(encoding="utf-8")
+    if hasattr(sys.stdin, "buffer"):
+        return sys.stdin.buffer.read().decode("utf-8")
+    return sys.stdin.read()
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="lede",
+        description="Deterministic extractive summarization.",
+    )
+    parser.add_argument(
+        "path",
+        nargs="?",
+        default=None,
+        help="Input file path. Reads stdin if omitted.",
+    )
+    parser.add_argument(
+        "--mode",
+        choices=["tfidf", "keyword", "clean_text", "strip_think"],
+        default="tfidf",
+        help="Summarization mode (default: tfidf).",
+    )
+    parser.add_argument(
+        "--max-chars",
+        type=int,
+        default=500,
+        help="Character budget for tfidf mode (default: 500).",
+    )
+    parser.add_argument(
+        "--keywords",
+        default="",
+        help="Space-separated keywords for keyword mode.",
+    )
+    parser.add_argument(
+        "--top",
+        type=int,
+        default=10,
+        help="Number of sentences to return in keyword mode (default: 10).",
+    )
+    args = parser.parse_args(argv)
+
+    text = _read_input(args.path)
+
+    if args.mode == "tfidf":
+        output = summarize(text, max_length=args.max_chars).summary
+    elif args.mode == "keyword":
+        if not args.keywords:
+            parser.error("--mode keyword requires --keywords")
+        output = extract_keyword(text, args.keywords, num_sentences=args.top)
+    elif args.mode == "clean_text":
+        output = clean_text(text)
+    elif args.mode == "strip_think":
+        output = strip_think(text)
+    else:  # pragma: no cover
+        parser.error(f"unknown mode: {args.mode}")
+
+    sys.stdout.write(output)
+    if not output.endswith("\n"):
+        sys.stdout.write("\n")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())