second-read 0.1.1__py3-none-any.whl

secondread/agents/originality/agent.py

@@ -0,0 +1,277 @@
+"""The originality checker agent.
+
+Deterministic-first: sentences are sampled in code (``sentences``), each is
+searched as an exact phrase, and search snippets (optionally the fetched page
+of the most promising hit) are compared to the sentence in code
+(``similarity``). No LLM is called. Every finding is ``heuristic=True`` with
+``Severity.CONSIDER``: a match is a prompt to check attribution, not a verdict.
+
+Search results and pages are untrusted data: they are only compared and
+quoted as excerpts, never interpreted.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+
+from secondread.agents.originality.sentences import Sentence, sample_sentences
+from secondread.agents.originality.similarity import Match, same_url, similarity
+from secondread.contracts.errors import BudgetExceeded, FetchError, ProviderError
+from secondread.contracts.interfaces import ReviewContext, SearchResult
+from secondread.contracts.models import (
+    AgentName,
+    AgentResult,
+    Evidence,
+    Finding,
+    ProgressEvent,
+    ProgressStep,
+    Severity,
+    Usage,
+    utcnow,
+)
+
+log = logging.getLogger(__name__)
+
+DEFAULT_SAMPLE_SIZE = 8
+"""Sentences searched per document (further capped by the search budget)."""
+
+DEFAULT_THRESHOLD = 0.6
+"""Minimum similarity for a possible match."""
+
+FETCH_FLOOR_RATIO = 0.5
+"""Default ``fetch_floor`` as a share of ``threshold``: snippets scoring at least
+this (but under the threshold) may trigger a page fetch."""
+
+DEFAULT_RESULTS_PER_QUERY = 5
+MAX_QUERY_WORDS = 32
+MAX_EVIDENCE = 3
+
+_SEARCH_USAGE = Usage(search_calls=1)
+_FETCH_USAGE = Usage(fetch_calls=1)
+
+
+@dataclass
+class _Hit:
+    url: str
+    title: str
+    match: Match
+
+
+@dataclass
+class _Run:
+    findings: list[Finding] = field(default_factory=list)
+    usage: Usage = field(default_factory=Usage)
+    checked: int = 0
+    own_skipped: int = 0
+    fetch_failed: int = 0
+    error: str | None = None
+
+
+def build_query(sentence: Sentence) -> str:
+    """Exact-phrase query for a sentence (first ``MAX_QUERY_WORDS`` words)."""
+    tokens = sentence.text.split()
+    phrase = " ".join(tokens[:MAX_QUERY_WORDS]).strip(" \"'“”‘’.,;:!?")
+    phrase = phrase.replace('"', "").replace("“", "").replace("”", "")
+    return f'"{phrase}"'
+
+
+class OriginalityAgent:
+    """Searches the web for near-copies of the document's most distinctive sentences.
+
+    Implements ``Agent`` with ``name == AgentName.ORIGINALITY``. Returns
+    findings only (no ledger updates). ``sample_size`` caps the sentences
+    searched, and never more than the searches left in the job budget.
+    With ``fetch_pages``, a hit whose snippet is suggestive (at least
+    ``fetch_floor``, default half the threshold) but under ``threshold`` is
+    fetched, once per sentence, and compared against the page text. Never
+    raises for provider or budget failures: returns partial findings with ``AgentResult.error`` set.
+    """
+
+    def __init__(
+        self,
+        *,
+        sample_size: int = DEFAULT_SAMPLE_SIZE,
+        threshold: float = DEFAULT_THRESHOLD,
+        results_per_query: int = DEFAULT_RESULTS_PER_QUERY,
+        fetch_pages: bool = True,
+        fetch_floor: float | None = None,
+    ) -> None:
+        if sample_size < 0 or results_per_query <= 0:
+            raise ValueError("sample_size must be >= 0 and results_per_query > 0")
+        if fetch_floor is None:
+            fetch_floor = threshold * FETCH_FLOOR_RATIO
+        if not 0.0 < threshold <= 1.0 or not 0.0 <= fetch_floor <= threshold:
+            raise ValueError("need 0 < threshold <= 1 and 0 <= fetch_floor <= threshold")
+        self.sample_size = sample_size
+        self.threshold = threshold
+        self.results_per_query = results_per_query
+        self.fetch_pages = fetch_pages
+        self.fetch_floor = fetch_floor
+
+    @property
+    def name(self) -> str:
+        """``AgentName.ORIGINALITY``."""
+        return AgentName.ORIGINALITY
+
+    def sample(self, ctx: ReviewContext) -> tuple[list[Sentence], int]:
+        """``(sentences to search, sentences wanted)``: off-topic and distinctive ones.
+
+        ``wanted`` is capped by ``sample_size``; the returned sample is further
+        capped by the searches left in the job budget.
+        """
+        candidates = sample_sentences(ctx.document.text, self.sample_size)
+        wanted = len(candidates)
+        return candidates[: min(wanted, ctx.meter.searches_left())], wanted
+
+    async def run(self, ctx: ReviewContext) -> AgentResult:
+        """Sample, search and compare; return heuristic findings."""
+        sample, wanted = self.sample(ctx)
+        state = _Run()
+        if wanted and not sample:
+            state.error = "originality check skipped: search budget exhausted"
+        self._emit(ctx, f"Checking originality of {len(sample)} sentence(s)", 0.0)
+        for i, sentence in enumerate(sample):
+            self._emit(
+                ctx,
+                f"Checking originality of sentence {i + 1} of {len(sample)}",
+                100.0 * i / max(1, len(sample)),
+            )
+            if not await self._check(ctx, sentence, state):
+                break
+        notes = self._notes(state, wanted)
+        self._emit(ctx, f"Originality: {len(state.findings)} possible match(es)", 100.0)
+        log.info(
+            "originality: %d checked, %d findings, error=%s",
+            state.checked,
+            len(state.findings),
+            bool(state.error),
+        )
+        return AgentResult(
+            agent=self.name,
+            findings=state.findings,
+            usage=state.usage,
+            error=state.error,
+            notes=notes,
+        )
+
+    async def _check(self, ctx: ReviewContext, sentence: Sentence, state: _Run) -> bool:
+        """Check one sentence. Returns False when the run must stop."""
+        try:
+            ctx.meter.require_search()
+        except BudgetExceeded as exc:
+            state.error = f"budget exhausted: {exc}"
+            return False
+        try:
+            results = await ctx.search.search(build_query(sentence), k=self.results_per_query)
+        except ProviderError as exc:
+            state.error = f"originality search failed: {exc}"
+            return False
+        finally:
+            ctx.meter.charge(_SEARCH_USAGE)
+            state.usage = state.usage + _SEARCH_USAGE
+        state.checked += 1
+
+        hits, to_fetch = self._score_results(ctx, sentence, results, state)
+        if not hits and to_fetch is not None and self.fetch_pages:
+            try:
+                ctx.meter.require_time()
+            except BudgetExceeded as exc:
+                state.error = f"budget exhausted: {exc}"
+                return False
+            hit = await self._fetch_and_compare(ctx, sentence, to_fetch, state)
+            if hit is not None:
+                hits.append(hit)
+        if hits:
+            state.findings.append(self._finding(sentence, hits))
+        return True
+
+    def _score_results(
+        self,
+        ctx: ReviewContext,
+        sentence: Sentence,
+        results: list[SearchResult],
+        state: _Run,
+    ) -> tuple[list[_Hit], SearchResult | None]:
+        hits: list[_Hit] = []
+        seen: set[str] = set()
+        best_near: tuple[float, SearchResult] | None = None
+        for result in results:
+            if same_url(result.url, ctx.document.source_url):
+                state.own_skipped += 1
+                continue
+            if any(same_url(result.url, u) for u in seen):
+                continue
+            seen.add(result.url)
+            match = max(
+                similarity(sentence.text, result.snippet),
+                similarity(sentence.text, result.title),
+                key=lambda m: m.score,
+            )
+            if match.score >= self.threshold:
+                hits.append(_Hit(result.url, result.title, match))
+            elif match.score >= self.fetch_floor and (
+                best_near is None or match.score > best_near[0]
+            ):
+                best_near = (match.score, result)
+        return hits, (best_near[1] if best_near else None)
+
+    async def _fetch_and_compare(
+        self, ctx: ReviewContext, sentence: Sentence, result: SearchResult, state: _Run
+    ) -> _Hit | None:
+        try:
+            page = await ctx.fetcher.fetch(result.url)
+        except FetchError:
+            state.fetch_failed += 1
+            return None
+        finally:
+            ctx.meter.charge(_FETCH_USAGE)
+            state.usage = state.usage + _FETCH_USAGE
+        if same_url(page.final_url, ctx.document.source_url):
+            state.own_skipped += 1
+            return None
+        match = similarity(sentence.text, page.text)
+        if match.score < self.threshold:
+            return None
+        return _Hit(result.url, page.title or result.title, match)
+
+    def _finding(self, sentence: Sentence, hits: list[_Hit]) -> Finding:
+        hits = sorted(hits, key=lambda h: (-h.match.score, h.url))[:MAX_EVIDENCE]
+        top = hits[0].match.score
+        now = utcnow()
+        kind = "near-verbatim" if top >= 0.95 else "close"
+        others = f" and {len(hits) - 1} other source(s)" if len(hits) > 1 else ""
+        return Finding(
+            agent=self.name,
+            severity=Severity.CONSIDER,
+            span=sentence.span,
+            message=(
+                f"Possible {kind} match online (similarity {top:.2f}) at "
+                f"{hits[0].url}{others}. Heuristic check, not proof of copying."
+            ),
+            evidence=[
+                Evidence(url=h.url, title=h.title, excerpt=h.match.excerpt, retrieved_at=now)
+                for h in hits
+            ],
+            suggestion="If this wording comes from the source, quote and cite it.",
+            heuristic=True,
+        )
+
+    def _notes(self, state: _Run, wanted: int) -> list[str]:
+        notes: list[str] = []
+        unchecked = wanted - state.checked
+        if unchecked > 0:
+            notes.append(f"{unchecked} sampled sentence(s) not checked for originality")
+        if state.own_skipped:
+            notes.append(f"{state.own_skipped} result(s) from the document's own URL ignored")
+        if state.fetch_failed:
+            notes.append(f"{state.fetch_failed} page(s) could not be fetched")
+        return notes
+
+    def _emit(self, ctx: ReviewContext, message: str, percent: float) -> None:
+        try:
+            ctx.emit_progress(
+                ProgressEvent(step=ProgressStep.REVIEWING, message=message, percent=percent)
+            )
+        except Exception:  # progress must never break the check
+            log.warning("originality: progress callback raised", exc_info=True)

secondread/agents/originality/sentences.py

@@ -0,0 +1,311 @@
+"""Deterministic sampling of distinctive sentences.
+
+The document is split into prose sentences with exact character spans
+(``doc.text[span.start:span.end] == sentence.text``). Headings, code blocks,
+block quotes, table rows, sentences that mostly quote someone else and common
+boilerplate are skipped. The rest are scored for distinctiveness (long and
+heavy in uncommon words) and ranked; ties break by position, so the same
+document always yields the same sample.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterator
+from dataclasses import dataclass
+
+from secondread.contracts.models import Span
+
+MIN_WORDS = 8
+"""Sentences with fewer words are too short to be distinctive."""
+
+MIN_UNCOMMON_WORDS = 4
+"""Minimum distinct uncommon words for a sentence to qualify."""
+
+MIN_UNCOMMON_RATIO = 0.3
+"""Minimum share of uncommon words among all words."""
+
+MAX_QUOTED_RATIO = 0.5
+"""Sentences with more than this share of characters inside quotes are skipped."""
+
+_COMMON_WORDS_TEXT = """
+    a about above after again against all also although always am an and any are
+    around as at away back be because been before being below between both but by
+    can could did do does doing done down during each either else even ever every
+    few first for from further get gets getting go goes going good got great had has
+    have having he her here hers herself him himself his how however i if in into is
+    it its itself just know last less like little long made make makes many may me
+    might more most much must my myself never new next no nor not now of off often
+    on once one only or other others our ours ourselves out over own people really
+    right same say says said see seem seems she should since so some something still
+    such take than that the their theirs them themselves then there these they thing
+    things think this those though through thus time to too two under until up upon
+    us use used very want was way we well were what when where whether which while
+    who whom whose why will with within without would yes yet you your yours
+    yourself yourselves year years day days today
+"""
+COMMON_WORDS: frozenset[str] = frozenset(_COMMON_WORDS_TEXT.split())
+"""Very common English words; everything else counts as uncommon."""
+
+_BOILERPLATE = re.compile(
+    r"all rights reserved|copyright|\(c\)\s*\d{4}|©|subscribe|sign up|newsletter|"
+    r"click here|read more|table of contents|cookie|privacy policy|terms of (?:use|service)|"
+    r"share this|follow us|posted (?:on|by)|last updated|this article (?:was|has been)",
+    re.IGNORECASE,
+)
+
+_WORD = re.compile(r"[^\W_]+(?:['’][^\W_]+)*")
+_SENTENCE_END = re.compile(r"[.!?]+[\"'”’)\]]*(?=\s|$)")
+_BULLET = re.compile(r"^\s*(?:[-*+•]|\d{1,3}[.)])\s+")
+_TABLE_OR_RULE = re.compile(r"^\s*(?:\||[-=*_]{3,}\s*$)")
+_ABBREVIATIONS_TEXT = (
+    "mr mrs ms dr prof sr jr st vs etc e.g i.e cf fig no inc ltd co corp approx dept est"
+)
+_ABBREVIATIONS = frozenset(_ABBREVIATIONS_TEXT.split())
+_QUOTED = re.compile(r"\"[^\"]*\"|“[^”]*”")
+
+
+@dataclass(frozen=True)
+class Sentence:
+    """A candidate sentence with its exact span and distinctiveness score."""
+
+    span: Span
+    text: str
+    word_count: int
+    uncommon: int
+    score: float
+
+
+def words(text: str) -> list[str]:
+    """Lower-cased word tokens of ``text``."""
+    return [m.group(0).lower() for m in _WORD.finditer(text)]
+
+
+def is_uncommon(word: str) -> bool:
+    """True for words that are not very common (numbers always count)."""
+    if any(ch.isdigit() for ch in word):
+        return True
+    return len(word) >= 4 and word.replace("’", "'") not in COMMON_WORDS
+
+
+def _prose_blocks(text: str) -> Iterator[tuple[int, int]]:
+    """Yield ``(start, end)`` of prose paragraphs, skipping non-prose lines.
+
+    Headings, fenced code, indented code, block quotes, table rows and
+    horizontal rules end the current paragraph and are excluded.
+    """
+    block_start: int | None = None
+    block_end = 0
+    in_fence = False
+    pos = 0
+    for line in text.splitlines(keepends=True):
+        line_start, pos = pos, pos + len(line)
+        body = line.rstrip("\r\n")
+        stripped = body.strip()
+        if stripped.startswith(("```", "~~~")):
+            in_fence = not in_fence
+            skip = True
+        else:
+            skip = (
+                in_fence
+                or not stripped
+                or stripped.startswith(("#", ">"))
+                or body.startswith(("    ", "\t"))
+                or bool(_TABLE_OR_RULE.match(body))
+            )
+        if skip:
+            if block_start is not None:
+                yield block_start, block_end
+                block_start = None
+            continue
+        if _BULLET.match(body) and block_start is not None:
+            # Each list item is its own paragraph.
+            yield block_start, block_end
+            block_start = None
+        if block_start is None:
+            block_start = line_start
+        block_end = line_start + len(body)
+    if block_start is not None:
+        yield block_start, block_end
+
+
+def _is_boundary(block: str, match: re.Match[str]) -> bool:
+    before = block[: match.start()]
+    last = before.rsplit(None, 1)[-1] if before.split() else ""
+    if match.group(0).startswith(".") and last.lower().rstrip(".") in _ABBREVIATIONS:
+        return False
+    if len(last) == 1 and last.isupper() and match.group(0) == ".":
+        return False  # an initial, e.g. "J. Smith"
+    rest = block[match.end() :].lstrip()
+    return not rest or not rest[0].islower()
+
+
+def split_sentences(text: str) -> list[tuple[Span, str]]:
+    """Split the prose of ``text`` into sentences with exact spans.
+
+    Leading list markers and surrounding whitespace are excluded from spans.
+    """
+    out: list[tuple[Span, str]] = []
+    for b_start, b_end in _prose_blocks(text):
+        block = text[b_start:b_end]
+        pieces: list[tuple[int, int]] = []
+        prev = 0
+        for match in _SENTENCE_END.finditer(block):
+            if _is_boundary(block, match):
+                pieces.append((prev, match.end()))
+                prev = match.end()
+        if prev < len(block):
+            pieces.append((prev, len(block)))
+        for p_start, p_end in pieces:
+            piece = block[p_start:p_end]
+            bullet = _BULLET.match(piece)
+            lead = bullet.end() if bullet else len(piece) - len(piece.lstrip())
+            trimmed = piece[lead:].rstrip()
+            if not trimmed:
+                continue
+            start = b_start + p_start + lead
+            span = Span(start=start, end=start + len(trimmed))
+            out.append((span, trimmed))
+    return out
+
+
+def quoted_ratio(text: str) -> float:
+    """Share of characters of ``text`` inside double quotes."""
+    if not text:
+        return 0.0
+    return sum(len(m.group(0)) for m in _QUOTED.finditer(text)) / len(text)
+
+
+def score_sentence(span: Span, text: str) -> Sentence | None:
+    """Score one sentence, or None if it is not a useful sample."""
+    tokens = words(text)
+    if len(tokens) < MIN_WORDS:
+        return None
+    if quoted_ratio(text) > MAX_QUOTED_RATIO or _BOILERPLATE.search(text):
+        return None
+    uncommon = {t for t in tokens if is_uncommon(t)}
+    ratio = sum(1 for t in tokens if is_uncommon(t)) / len(tokens)
+    if len(uncommon) < MIN_UNCOMMON_WORDS or ratio < MIN_UNCOMMON_RATIO:
+        return None
+    # Longer sentences with more distinct uncommon words are more
+    # fingerprint-like; cap length so run-ons do not dominate.
+    score = len(uncommon) * (0.5 + ratio) + min(len(tokens), 40) * 0.1
+    return Sentence(
+        span=span, text=text, word_count=len(tokens), uncommon=len(uncommon), score=score
+    )
+
+
+def distinctive_sentences(text: str) -> list[Sentence]:
+    """All qualifying sentences, most distinctive first (ties by position).
+
+    Sentences with identical wording are kept once (the first occurrence).
+    """
+    seen: set[tuple[str, ...]] = set()
+    ranked: list[Sentence] = []
+    for span, sentence in split_sentences(text):
+        scored = score_sentence(span, sentence)
+        if scored is None:
+            continue
+        key = tuple(words(sentence))
+        if key in seen:
+            continue
+        seen.add(key)
+        ranked.append(scored)
+    ranked.sort(key=lambda s: (-s.score, s.span.start))
+    return ranked
+
+
+# ---------------------------------------------------------------------------
+# Off-topic sentences
+# ---------------------------------------------------------------------------
+
+OFF_TOPIC_MIN_WORDS = 6
+"""Shortest sentence considered as an off-topic candidate."""
+
+OFF_TOPIC_MIN_UNCOMMON = 2
+"""Minimum distinct uncommon words for an off-topic candidate."""
+
+OFF_TOPIC_MAX_OVERLAP = 0.5
+"""Candidates sharing more than this share of topic words with the rest of the
+document are on-topic and not sampled as off-topic."""
+
+
+def _stem(word: str) -> str:
+    """Crude stem so ``battles``/``battle`` and ``wins``/``winning`` match."""
+    word = word.replace("’", "'").split("'")[0]
+    for suffix in ("ing", "ies", "ed", "es", "s"):
+        if len(word) - len(suffix) >= 4 and word.endswith(suffix):
+            return word[: -len(suffix)]
+    return word
+
+
+def _topic_stems(text: str) -> set[str]:
+    return {_stem(t) for t in words(text) if is_uncommon(t) and not t[0].isdigit()}
+
+
+def off_topic_sentences(text: str) -> list[Sentence]:
+    """Sentences that share few topic words with the rest of the document.
+
+    A borrowed line (a proverb, a famous quotation, a passage lifted from
+    elsewhere) often talks about something other than the article around it
+    and is too short or too plainly worded to rank as "distinctive". These are
+    ranked least-related first (ties: longer first, then position). ``score``
+    is ``1 - overlap``.
+    """
+    sentences = [
+        (span, sentence, _topic_stems(sentence)) for span, sentence in split_sentences(text)
+    ]
+    counts: dict[str, int] = {}
+    for _span, _sentence, stems in sentences:
+        for stem in stems:
+            counts[stem] = counts.get(stem, 0) + 1
+    seen: set[tuple[str, ...]] = set()
+    ranked: list[Sentence] = []
+    for span, sentence, stems in sentences:
+        tokens = words(sentence)
+        if len(tokens) < OFF_TOPIC_MIN_WORDS or len(stems) < OFF_TOPIC_MIN_UNCOMMON:
+            continue
+        if quoted_ratio(sentence) > MAX_QUOTED_RATIO or _BOILERPLATE.search(sentence):
+            continue
+        key = tuple(tokens)
+        if key in seen:
+            continue
+        seen.add(key)
+        shared = sum(1 for stem in stems if counts[stem] > 1)
+        overlap = shared / len(stems)
+        if overlap > OFF_TOPIC_MAX_OVERLAP:
+            continue
+        ranked.append(
+            Sentence(
+                span=span,
+                text=sentence,
+                word_count=len(tokens),
+                uncommon=len(stems),
+                score=1.0 - overlap,
+            )
+        )
+    ranked.sort(key=lambda s: (-s.score, -s.word_count, s.span.start))
+    return ranked
+
+
+def sample_sentences(text: str, limit: int) -> list[Sentence]:
+    """Up to ``limit`` sentences to check, alternating the two rankings.
+
+    Takes the most distinctive sentence, then the least on-topic, and so on,
+    skipping repeats, so both a lifted aphorism and a lifted dense passage
+    are likely to be sampled. Deterministic for a given text.
+    """
+    pools = [distinctive_sentences(text), off_topic_sentences(text)]
+    chosen: list[Sentence] = []
+    taken: set[Span] = set()
+    indexes = [0, 0]
+    while len(chosen) < limit and any(indexes[i] < len(pools[i]) for i in (0, 1)):
+        for i in (0, 1):
+            while indexes[i] < len(pools[i]) and pools[i][indexes[i]].span in taken:
+                indexes[i] += 1
+            if indexes[i] < len(pools[i]) and len(chosen) < limit:
+                sentence = pools[i][indexes[i]]
+                indexes[i] += 1
+                taken.add(sentence.span)
+                chosen.append(sentence)
+    return chosen

secondread/agents/originality/similarity.py

@@ -0,0 +1,114 @@
+"""Near-match detection between a sentence and untrusted web text.
+
+The score is the share of the sentence's words that fall inside runs of at
+least ``min_run`` consecutive words also found, in order, in the other text
+(``difflib.SequenceMatcher`` over word tokens, case- and punctuation-
+insensitive). 1.0 means the sentence appears verbatim; light edits keep the
+score high; a shared topic with different wording scores near 0.
+
+For long texts (fetched pages) the comparison runs on the window with the most
+shared word trigrams, which keeps it fast and locates the excerpt.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from difflib import SequenceMatcher
+from urllib.parse import urlsplit
+
+_WORD = re.compile(r"[^\W_]+(?:['’][^\W_]+)*")
+
+MAX_EXCERPT_CHARS = 300
+"""Excerpts longer than this are truncated with an ellipsis."""
+
+
+@dataclass(frozen=True)
+class _Token:
+    word: str
+    start: int
+    end: int
+
+
+@dataclass(frozen=True)
+class Match:
+    """Similarity of a sentence to some text, with the matching excerpt."""
+
+    score: float
+    excerpt: str
+
+
+def _tokens(text: str) -> list[_Token]:
+    return [
+        _Token(m.group(0).lower().replace("’", "'"), m.start(), m.end())
+        for m in _WORD.finditer(text)
+    ]
+
+
+def _best_window(sentence: list[str], other: list[str], size: int, n: int) -> tuple[int, int]:
+    """``(start, end)`` of the ``size``-token window of ``other`` sharing most n-grams."""
+    grams = {tuple(sentence[i : i + n]) for i in range(len(sentence) - n + 1)}
+    hits = [i for i in range(len(other) - n + 1) if tuple(other[i : i + n]) in grams]
+    if not hits:
+        return 0, 0
+    best_start, best_count, lo = hits[0], 0, 0
+    for hi, pos in enumerate(hits):
+        while pos - hits[lo] >= size:
+            lo += 1
+        if hi - lo + 1 > best_count:
+            best_count, best_start = hi - lo + 1, hits[lo]
+    start = max(0, best_start - len(sentence) // 2)
+    return start, min(len(other), start + size)
+
+
+def similarity(
+    sentence: str, other: str, *, min_run: int = 3, max_direct_tokens: int = 400
+) -> Match:
+    """Score how closely ``other`` reproduces ``sentence`` (0.0 to 1.0)."""
+    s_tokens = _tokens(sentence)
+    o_tokens = _tokens(other)
+    if not s_tokens or not o_tokens:
+        return Match(0.0, "")
+    s_words = [t.word for t in s_tokens]
+    run = max(1, min(min_run, len(s_words)))
+    if len(o_tokens) > max_direct_tokens:
+        size = 2 * len(s_words) + 10
+        lo, hi = _best_window(s_words, [t.word for t in o_tokens], size, run)
+        o_tokens = o_tokens[lo:hi]
+        if not o_tokens:
+            return Match(0.0, "")
+    o_words = [t.word for t in o_tokens]
+    matcher = SequenceMatcher(None, s_words, o_words, autojunk=False)
+    blocks = [b for b in matcher.get_matching_blocks() if b.size >= run]
+    if not blocks:
+        return Match(0.0, "")
+    matched = sum(b.size for b in blocks)
+    first, last = blocks[0], blocks[-1]
+    start = o_tokens[first.b].start
+    end = o_tokens[last.b + last.size - 1].end
+    return Match(round(matched / len(s_words), 4), _truncate(other[start:end]))
+
+
+def _truncate(text: str) -> str:
+    text = " ".join(text.split())
+    if len(text) <= MAX_EXCERPT_CHARS:
+        return text
+    return text[: MAX_EXCERPT_CHARS - 1].rstrip() + "…"
+
+
+def normalize_url(url: str) -> str:
+    """Comparable form of a URL: no scheme, ``www.``, fragment or trailing slash."""
+    parts = urlsplit(url.strip())
+    host = (parts.hostname or "").lower().removeprefix("www.")
+    if not host:
+        return url.strip().lower().rstrip("/")
+    path = parts.path.rstrip("/")
+    query = f"?{parts.query}" if parts.query else ""
+    return f"{host}{path}{query}"
+
+
+def same_url(a: str | None, b: str | None) -> bool:
+    """True if two URLs point at the same page (by ``normalize_url``)."""
+    if not a or not b:
+        return False
+    return normalize_url(a) == normalize_url(b)