unique_web_search 2026.44.0.dev4__tar.gz → 2026.44.0.dev5__tar.gz

{unique_web_search-2026.44.0.dev4 → unique_web_search-2026.44.0.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: unique-web-search
-Version: 2026.44.0.dev4
+Version: 2026.44.0.dev5
 Summary: 
 Author: Andreas Hauri, Gustav Hartz, Rami Azouz
 Author-email: Andreas Hauri <andreas@unique.ch>, Gustav Hartz <gustav.hartz.ext@unique.ch>, Rami Azouz <rami.ext@unique.ch>
@@ -16,7 +16,7 @@ Requires-Dist: tavily-python>=0.7.11,<1
 Requires-Dist: unidecode>=1.4.0,<2
 Requires-Dist: azure-ai-projects>=2.3,<3
 Requires-Dist: azure-identity>=1.25.0,<2
-Requires-Dist: unique-toolkit[monitoring]>=2026.44.0.dev9,<2026.44.0rc0
+Requires-Dist: unique-toolkit[monitoring]>=2026.44.0.dev10,<2026.44.0rc0
 Requires-Dist: azure-core>=1.36.0,<2
 Requires-Dist: google-cloud-aiplatform>=1.128.0,<2
 Requires-Dist: google-auth>=2.43.0,<3
@@ -25,8 +25,8 @@ Requires-Dist: rapidfuzz>=3.0.0,<4
 Requires-Dist: httpx>=0.28.1
 Requires-Dist: certifi>=2025.11.12
 Requires-Dist: jinja2>=3.1.6
-Requires-Dist: unique-search-proxy-core>=2026.44.0.dev6,<2026.44.0rc0
-Requires-Dist: unique-search-proxy-sdk>=2026.44.0.dev3,<2026.44.0rc0
+Requires-Dist: unique-search-proxy-core>=2026.44.0.dev10,<2026.44.0rc0
+Requires-Dist: unique-search-proxy-sdk>=2026.44.0.dev5,<2026.44.0rc0
 Requires-Dist: openai>=1.105.0,<3
 Requires-Python: >=3.12, <4.0
 Description-Content-Type: text/markdown
@@ -73,7 +73,7 @@ Standard engines that live in proxy-core use **`ExposableParam`** (`expose` + `v
 
 ### Agent search (grounding)
 
-An upstream AI agent / grounded model searches and returns opaque text that this tool parses into `WebSearchResult`s.
+An upstream AI agent / grounded model searches and returns opaque text that this tool parses into `WebSearchResult`s. Bing results take their URLs only from the citations Foundry attaches to the answer, never from the answer text.
 
 | Engine | Proxy | Notes |
 |--------|-------|-------|

{unique_web_search-2026.44.0.dev4 → unique_web_search-2026.44.0.dev5}/README.md

@@ -40,7 +40,7 @@ Standard engines that live in proxy-core use **`ExposableParam`** (`expose` + `v
 
 ### Agent search (grounding)
 
-An upstream AI agent / grounded model searches and returns opaque text that this tool parses into `WebSearchResult`s.
+An upstream AI agent / grounded model searches and returns opaque text that this tool parses into `WebSearchResult`s. Bing results take their URLs only from the citations Foundry attaches to the answer, never from the answer text.
 
 | Engine | Proxy | Notes |
 |--------|-------|-------|

{unique_web_search-2026.44.0.dev4 → unique_web_search-2026.44.0.dev5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "unique_web_search"
-version = "2026.44.0.dev4"
+version = "2026.44.0.dev5"
 description = ""
 readme = "README.md"
 requires-python = ">=3.12,<4.0"
@@ -22,7 +22,7 @@ dependencies = [
     "unidecode>=1.4.0,<2",
     "azure-ai-projects>=2.3,<3",
     "azure-identity>=1.25.0,<2",
-    "unique-toolkit[monitoring]>=2026.44.0.dev9,<2026.44.0rc0",
+    "unique-toolkit[monitoring]>=2026.44.0.dev10,<2026.44.0rc0",
     "azure-core>=1.36.0,<2",
     "google-cloud-aiplatform>=1.128.0,<2",
     "google-auth>=2.43.0,<3",
@@ -31,8 +31,8 @@ dependencies = [
     "httpx>=0.28.1",
     "certifi>=2025.11.12",
     "jinja2>=3.1.6",
-    "unique-search-proxy-core>=2026.44.0.dev6,<2026.44.0rc0",
-    "unique-search-proxy-sdk>=2026.44.0.dev3,<2026.44.0rc0",
+    "unique-search-proxy-core>=2026.44.0.dev10,<2026.44.0rc0",
+    "unique-search-proxy-sdk>=2026.44.0.dev5,<2026.44.0rc0",
     "openai>=1.105.0,<3",
 ]
 

{unique_web_search-2026.44.0.dev4 → unique_web_search-2026.44.0.dev5}/src/unique_web_search/metrics.py

@@ -15,6 +15,21 @@ custom_api_url_safety_report = m.counter(
     "Report-only URL safety outcomes for legacy Custom API searches",
     ["outcome", "reason_category"],
 )
+grounding_parse_failures = m.counter(
+    "grounding_parse_failures_total",
+    "Grounded agent answers that yielded no usable sources",
+    ["engine", "reason"],
+)
+grounding_dropped_citations = m.counter(
+    "grounding_dropped_citations_total",
+    "Citations dropped from grounded agent answers because they failed validation",
+    ["engine", "reason"],
+)
+grounding_dropped_lines = m.counter(
+    "grounding_dropped_lines_total",
+    "Answer lines dropped from grounded agent answers",
+    ["engine", "reason"],
+)
 
 # End-to-End Tool (P1)
 tool_duration = m.histogram(

{unique_web_search-2026.44.0.dev4 → unique_web_search-2026.44.0.dev5}/src/unique_web_search/services/proxy/mappers.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 from collections.abc import Sequence
 from typing import TYPE_CHECKING
 
+from unique_search_proxy_core.schema import AnswerSpanCitation
 from unique_search_proxy_sdk._generated.models.agent_search_response import (
     AgentSearchResponse,
 )
@@ -71,3 +72,12 @@ def agent_answer_text(response: AgentSearchResponse) -> str:
     if isinstance(response.answer, str):
         return response.answer
     return ""
+
+
+def agent_citations(response: AgentSearchResponse) -> list[AnswerSpanCitation]:
+    if not isinstance(response.citations, list):
+        return []
+    return [
+        AnswerSpanCitation.model_validate(citation.to_dict())
+        for citation in response.citations
+    ]

{unique_web_search-2026.44.0.dev4 → unique_web_search-2026.44.0.dev5}/src/unique_web_search/services/search_engine/README.md

@@ -31,7 +31,7 @@ Deployment configs for Google / Brave / Perplexity (and agent base fields) live
 | | Standard | Agent (grounding) |
 |--|----------|-------------------|
 | Proxy route | `POST /v1/search` | `POST /v1/agent-search` |
-| Result shape | `SearchResponse` → `WebSearchResult` list | Opaque `answer` text → parsed locally |
+| Result shape | `SearchResponse` → `WebSearchResult` list | Opaque `answer` text (+ provider `citations`) → parsed locally |
 | LLM knobs | `ExposableParam` + `config.merge()` | `ExposableParam` (Bing only) |
 | Typical scrape | Often yes (Google) | Often no (`requires_scraping=false`) |
 
@@ -158,7 +158,13 @@ CUSTOM_WEB_SEARCH_API_HEADERS='{"Authorization": "Bearer ..."}'
 
 ### Bing (Grounding with Bing)
 
-**Provider:** Azure AI Foundry Agent with Bing grounding · Config extends `BingAgentConfig` with `requires_scraping`, `language_model`, etc.
+**Provider:** Azure AI Foundry Agent with Bing grounding · Config extends `BingAgentConfig` with `requires_scraping`.
+
+The agent answers with one bullet per source, each ending in a citation marker.
+`BingCitationParser` (`utils/grounding/bing/citation_parsing.py`) turns each
+cited source into one result, taking its URL and title only from Foundry's
+`url_citation` annotations. Unusable citations and lines are dropped one by one
+and counted in the `grounding_*` metrics rather than failing the whole answer.
 
 | Field | Notes |
 |-------|-------|

{unique_web_search-2026.44.0.dev4 → unique_web_search-2026.44.0.dev5}/src/unique_web_search/services/search_engine/base.py

@@ -24,6 +24,7 @@ from unique_web_search.services.proxy.bridge import (
 )
 from unique_web_search.services.proxy.mappers import (
     agent_answer_text,
+    agent_citations,
     map_agent_answer,
     map_search_response,
 )
@@ -32,6 +33,10 @@ from unique_web_search.services.search_engine.schema import (
 )
 
 if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+    from unique_search_proxy_core.schema import AnswerSpanCitation
+
     from unique_web_search.services.search_engine.utils.grounding.response_parsing import (
         ResponseParser,
     )
@@ -72,7 +77,7 @@ class SearchEngine(ABC, Generic[SearchEngineConfig]):
     it, otherwise it reroutes to the local legacy implementation.
     """
 
-    # Set by agent engines (Bing/VertexAI) and consumed by the agent proxy path.
+    # Set by agent engines that use the default ``_parse_agent_answer``.
     response_parsers: list[ResponseParser]
 
     def __init__(
@@ -180,12 +185,24 @@ class SearchEngine(ABC, Generic[SearchEngineConfig]):
                 engine=engine.value,
                 **invocation,
             )
-        results = await map_agent_answer(
+        results = await self._parse_agent_answer(
             agent_answer_text(response),
-            self.response_parsers,
+            agent_citations(response),
         )
         return await self._postprocess_results(results)
 
+    async def _parse_agent_answer(
+        self,
+        answer: str,
+        citations: Sequence[AnswerSpanCitation],
+    ) -> list[WebSearchResult]:
+        """Turn an agent answer into results.
+
+        Parses only the answer text. Engines whose provider cites its sources
+        alongside the answer override this, as Bing does.
+        """
+        return await map_agent_answer(answer, self.response_parsers)
+
     def _agent_proxy_invocation(
         self,
         engine: AgentEngineType,

{unique_web_search-2026.44.0.dev4 → unique_web_search-2026.44.0.dev5}/src/unique_web_search/services/search_engine/bing.py

@@ -1,4 +1,5 @@
 import asyncio
+from collections.abc import Sequence
 from typing import Any, override
 
 from pydantic import Field
@@ -10,10 +11,13 @@ from unique_search_proxy_core.agent_engines.bing.schema import BingAgentConfig
 from unique_search_proxy_core.agent_engines.bing.settings import resolve_market
 from unique_search_proxy_core.context import LOCAL_REQUEST_CONTEXT, RequestContext
 from unique_search_proxy_core.param_policy.exposed_params import ExposedParams
-from unique_toolkit._common.default_language_model import DEFAULT_LANGUAGE_MODEL
-from unique_toolkit._common.validators import LMI, get_LMI_default_field
-from unique_toolkit.language_model import LanguageModelService
+from unique_search_proxy_core.schema import AnswerSpanCitation
 
+from unique_web_search.metrics import (
+    grounding_dropped_citations,
+    grounding_dropped_lines,
+    grounding_parse_failures,
+)
 from unique_web_search.services.proxy.bridge import (
     search_proxy_client_enabled,
 )
@@ -22,11 +26,8 @@ from unique_web_search.services.search_engine.registry import register_search_en
 from unique_web_search.services.search_engine.schema import (
     WebSearchResult,
 )
-from unique_web_search.services.search_engine.utils.grounding import (
-    JsonConversionStrategy,
-    LLMParserStrategy,
-)
 from unique_web_search.services.search_engine.utils.grounding.bing import (
+    BingCitationParser,
     create_and_process_run,
     credentials_are_valid,
     get_credentials,
@@ -35,10 +36,6 @@ from unique_web_search.services.search_engine.utils.grounding.bing import (
 
 
 class BingSearchConfig(BingAgentConfig):
-    language_model: LMI = get_LMI_default_field(
-        DEFAULT_LANGUAGE_MODEL,
-        description="The language model to use in as a fallback parser if the agent response is not a valid JSON.",
-    )
     requires_scraping: bool = Field(
         default=False,
         description="Whether the search engine requires scraping.",
@@ -51,26 +48,17 @@ class BingSearchConfig(BingAgentConfig):
     config_cls=BingSearchConfig,
     mode=SearchEngineMode.AGENT,
     config_display_name="Grounding with Bing",
-    needs_language_model=True,
 )
 class BingSearch(SearchEngine[BingSearchConfig]):
     def __init__(
         self,
         config: BingSearchConfig,
-        language_model_service: LanguageModelService,
         *,
         request_context: RequestContext = LOCAL_REQUEST_CONTEXT,
     ):
         super().__init__(config, request_context=request_context)
         self.credentials = get_credentials()
-
-        self.response_parsers = [
-            JsonConversionStrategy(),
-            LLMParserStrategy(
-                config.language_model,
-                language_model_service,
-            ),
-        ]
+        self.citation_response_parser = BingCitationParser()
 
     @property
     def is_configured(self) -> bool:
@@ -116,12 +104,33 @@ class BingSearch(SearchEngine[BingSearchConfig]):
         agent_client = get_project_client(self.credentials)
 
         async with agent_client:
-            search_results = await create_and_process_run(
+            answer, citations = await create_and_process_run(
                 agent_client,
                 query=query,
                 grounding=self._grounding_configuration(),
-                response_parsers_strategies=self.response_parsers,
                 generation_instructions=self.config.generation_instructions,
             )
 
-        return search_results
+        return await self._parse_agent_answer(answer, citations)
+
+    @override
+    async def _parse_agent_answer(
+        self,
+        answer: str,
+        citations: Sequence[AnswerSpanCitation],
+    ) -> list[WebSearchResult]:
+        parsed = await self.citation_response_parser(answer, citations)
+        engine = self.config.engine.value
+        for reason, count in parsed.dropped_citations.items():
+            grounding_dropped_citations.labels(engine=engine, reason=reason.value).inc(
+                count
+            )
+        for reason, count in parsed.dropped_lines.items():
+            grounding_dropped_lines.labels(engine=engine, reason=reason.value).inc(
+                count
+            )
+        if parsed.failure is not None:
+            grounding_parse_failures.labels(
+                engine=engine, reason=parsed.failure.value
+            ).inc()
+        return parsed.results

{unique_web_search-2026.44.0.dev4 → unique_web_search-2026.44.0.dev5}/src/unique_web_search/services/search_engine/utils/grounding/bing/__init__.py

@@ -1,3 +1,6 @@
+from unique_web_search.services.search_engine.utils.grounding.bing.citation_parsing import (
+    BingCitationParser,
+)
 from unique_web_search.services.search_engine.utils.grounding.bing.client import (
     credentials_are_valid,
     get_credentials,
@@ -8,6 +11,7 @@ from unique_web_search.services.search_engine.utils.grounding.bing.runner import
 )
 
 __all__ = [
+    "BingCitationParser",
     "credentials_are_valid",
     "get_credentials",
     "get_project_client",

unique_web_search-2026.44.0.dev5/src/unique_web_search/services/search_engine/utils/grounding/bing/citation_instructions.j2

@@ -0,0 +1,18 @@
+{{ generation_instructions }}
+
+## Source-isolated evidence list
+Use Bing Grounding to select distinct useful sources.
+The complete response must consist only of lines in this form:
+- <a paragraph of facts supported only by one source> <that source's citation>
+
+Each line is a paragraph, not a sentence. Write several sentences per line that report every relevant fact, figure, date, name and quote from that source, with as much detail as the source offers, often 100 to 200 words. Lines can be long; do not shorten them.
+
+Hard requirements:
+1. Exactly one line per distinct source and exactly one distinct source per line.
+2. Every line begins with `- ` and contains exactly one citation marker as its final content.
+3. Never synthesize, compare, or combine facts across sources within a line.
+4. Put all useful facts from a source into that source's one line; never cite it again.
+5. No headings, preface, conclusion, line breaks inside a paragraph, nested bullets, JSON, or uncited claims.
+6. Never invent, reconstruct, type, or guess citation markers or source URLs.
+
+Before responding, silently verify that the number of lines equals the number of distinct cited sources.

unique_web_search-2026.44.0.dev5/src/unique_web_search/services/search_engine/utils/grounding/bing/citation_parsing.py

@@ -0,0 +1,302 @@
+import logging
+import re
+from collections import Counter
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+from enum import StrEnum
+from urllib.parse import urlparse
+
+from unique_search_proxy_core.schema import AnswerSpanCitation
+
+from unique_web_search.services.search_engine.schema import WebSearchResult
+
+_LOGGER = logging.getLogger(__name__)
+_LIST_MARKER_PATTERN = re.compile(r"\s*(?:[-*+•]|\d+[.)])\s+")
+
+
+class GroundingCitationDropReason(StrEnum):
+    """Why one citation was left out of the results."""
+
+    INVALID_OFFSETS = "invalid_offsets"
+    MARKER_MISMATCH = "marker_mismatch"
+    NOT_ON_ONE_LINE = "citation_not_on_one_line"
+    NON_HTTP_URL = "non_http_url"
+
+
+class GroundingLineDropReason(StrEnum):
+    """Why one answer line was left out of the results."""
+
+    NO_CITATION = "no_citation"
+    NO_CONTENT = "no_content"
+
+
+class GroundingParseFailureReason(StrEnum):
+    """Why an answer produced no results at all."""
+
+    NO_TEXT = "no_text"
+    NO_CITATIONS = "no_citations"
+    NO_USABLE_CITATIONS = "no_usable_citations"
+
+
+@dataclass(frozen=True)
+class CitationParseResult:
+    """Results mapped from a grounded answer, plus the parts that were dropped.
+
+    ``failure`` is set exactly when no result survived.
+    """
+
+    results: list[WebSearchResult]
+    dropped_citations: Counter[GroundingCitationDropReason] = field(
+        default_factory=Counter
+    )
+    dropped_lines: Counter[GroundingLineDropReason] = field(default_factory=Counter)
+    failure: GroundingParseFailureReason | None = None
+
+
+class BingCitationParser:
+    """Map a Grounding with Bing Search answer onto one result per cited source.
+
+    The agent is prompted to answer with one bullet per source, each ending in
+    the marker Foundry renders for that source. Foundry returns the answer text
+    and, separately, one citation per marker with the marker's character span
+    in the answer and the source it points to. For example, this answer::
+
+        - Python 3.14 adds template string literals (PEP 750). 【4:0†source】
+        - Free-threaded Python is officially supported. 【4:1†source】
+
+    comes with these citations::
+
+        AnswerSpanCitation(
+            url="https://docs.python.org/3/whatsnew/3.14.html",
+            title="What's new in Python 3.14",
+            marker="【4:0†source】",
+            start_index=55,
+            end_index=67,
+        )
+        AnswerSpanCitation(
+            url="https://peps.python.org/pep-0779/",
+            title="PEP 779",
+            marker="【4:1†source】",
+            start_index=116,
+            end_index=128,
+        )
+
+    and is parsed into one result per URL. A result's content is the text of
+    the lines citing it, without list markers or citation markers, such as
+    ``"Python 3.14 adds template string literals (PEP 750)."``. Result URLs and
+    titles come only from citations, never from the answer text.
+
+    The model does not always follow the format, so these deviations are
+    accepted:
+
+    - Other list markers (``*``, ``+``, ``•``, ``1.``, ``1)``), indentation, or
+      no list marker at all.
+    - A marker anywhere in the line; the text after it is kept.
+    - Several sources cited on one line; the line is added to each of them.
+    - One source cited on several lines; its lines are joined in order.
+
+    Whatever cannot be used is dropped on its own and counted, and the rest is
+    kept:
+
+    - A citation whose offsets fall outside the answer, whose span does not
+      hold its marker, whose span crosses a line break, or whose URL is not
+      http(s). A citation dropped only for its URL still has its marker
+      removed from the line text.
+    - A line with no usable citation, such as a heading or a preface, or with
+      no text left once its markers are removed.
+    """
+
+    async def __call__(
+        self,
+        response: str,
+        citations: Sequence[AnswerSpanCitation],
+    ) -> CitationParseResult:
+        lines = _answer_lines(response)
+        if not lines:
+            _LOGGER.warning("Grounding response contained no text")
+            return CitationParseResult(
+                results=[], failure=GroundingParseFailureReason.NO_TEXT
+            )
+        if not citations:
+            _LOGGER.warning("Grounding response contained no citations")
+            return CitationParseResult(
+                results=[], failure=GroundingParseFailureReason.NO_CITATIONS
+            )
+
+        cited_lines, dropped_citations = _assign_citations_to_lines(
+            response, lines, citations
+        )
+        sources, dropped_lines = _group_lines_by_source(response, cited_lines)
+        results = [source.to_search_result() for source in sources]
+
+        if dropped_citations or dropped_lines:
+            _LOGGER.info(
+                "Grounding response parsed with drops result_count=%d "
+                "dropped_citations=%s dropped_lines=%s",
+                len(results),
+                {reason.value: count for reason, count in dropped_citations.items()},
+                {reason.value: count for reason, count in dropped_lines.items()},
+            )
+        if not results:
+            _LOGGER.warning("Grounding response had no usable citations")
+        return CitationParseResult(
+            results=results,
+            dropped_citations=dropped_citations,
+            dropped_lines=dropped_lines,
+            failure=(
+                None if results else GroundingParseFailureReason.NO_USABLE_CITATIONS
+            ),
+        )
+
+
+@dataclass(frozen=True)
+class _AnswerLine:
+    """A non-empty line of the answer, with its offsets into the answer."""
+
+    start: int
+    end: int
+    text: str
+
+    def contains(self, citation: AnswerSpanCitation) -> bool:
+        return self.start <= citation.start_index and citation.end_index <= self.end
+
+
+@dataclass
+class _CitedLine:
+    """An answer line and the verified citations whose markers are on it.
+
+    ``markers`` holds all of them, since every marker is removed from the text.
+    ``sources`` holds the ones whose URL can become a result.
+    """
+
+    line: _AnswerLine
+    markers: list[AnswerSpanCitation] = field(default_factory=list)
+    sources: list[AnswerSpanCitation] = field(default_factory=list)
+
+
+@dataclass
+class _Source:
+    """A cited URL and the content of every line that cites it."""
+
+    url: str
+    title: str
+    # Keyed by line index, so a line citing the same URL twice is added once.
+    contents: dict[int, str] = field(default_factory=dict)
+
+    def to_search_result(self) -> WebSearchResult:
+        content = "\n".join(self.contents.values())
+        return WebSearchResult(
+            url=self.url, title=self.title, snippet=content, content=content
+        )
+
+
+def _answer_lines(response: str) -> list[_AnswerLine]:
+    """Split the answer into its non-empty lines, keeping their offsets."""
+    lines: list[_AnswerLine] = []
+    offset = 0
+    for raw_line in response.splitlines(keepends=True):
+        text = raw_line.rstrip("\r\n")
+        if text.strip():
+            lines.append(_AnswerLine(start=offset, end=offset + len(text), text=text))
+        offset += len(raw_line)
+    return lines
+
+
+def _assign_citations_to_lines(
+    response: str,
+    lines: Sequence[_AnswerLine],
+    citations: Sequence[AnswerSpanCitation],
+) -> tuple[list[_CitedLine], Counter[GroundingCitationDropReason]]:
+    """Attach each verified citation to the line its marker is on.
+
+    Returns every line, cited or not, and the dropped citations counted by
+    reason.
+    """
+    cited_lines = [_CitedLine(line) for line in lines]
+    dropped: Counter[GroundingCitationDropReason] = Counter()
+    for citation in citations:
+        reason = _marker_drop_reason(response, citation)
+        if reason is not None:
+            dropped[reason] += 1
+            continue
+        cited_line = _line_containing(cited_lines, citation)
+        if cited_line is None:
+            dropped[GroundingCitationDropReason.NOT_ON_ONE_LINE] += 1
+            continue
+        cited_line.markers.append(citation)
+        if not _is_http_url(citation.url):
+            dropped[GroundingCitationDropReason.NON_HTTP_URL] += 1
+            continue
+        cited_line.sources.append(citation)
+    return cited_lines, dropped
+
+
+def _marker_drop_reason(
+    response: str, citation: AnswerSpanCitation
+) -> GroundingCitationDropReason | None:
+    """Return why the citation's span does not hold its marker, or ``None`` if it does."""
+    if not (0 <= citation.start_index < citation.end_index <= len(response)):
+        return GroundingCitationDropReason.INVALID_OFFSETS
+    if response[citation.start_index : citation.end_index] != citation.marker:
+        return GroundingCitationDropReason.MARKER_MISMATCH
+    return None
+
+
+def _line_containing(
+    cited_lines: Sequence[_CitedLine], citation: AnswerSpanCitation
+) -> _CitedLine | None:
+    """Return the line holding the citation's whole span, if there is one."""
+    return next((cited for cited in cited_lines if cited.line.contains(citation)), None)
+
+
+def _is_http_url(url: str) -> bool:
+    parsed_url = urlparse(url)
+    return parsed_url.scheme in {"http", "https"} and bool(parsed_url.netloc)
+
+
+def _group_lines_by_source(
+    response: str,
+    cited_lines: Sequence[_CitedLine],
+) -> tuple[list[_Source], Counter[GroundingLineDropReason]]:
+    """Add each line's content to every source it cites.
+
+    Returns the sources in the order they are first cited, and the dropped
+    lines counted by reason.
+    """
+    sources: dict[str, _Source] = {}
+    dropped: Counter[GroundingLineDropReason] = Counter()
+    for index, cited_line in enumerate(cited_lines):
+        if not cited_line.sources:
+            dropped[GroundingLineDropReason.NO_CITATION] += 1
+            continue
+        content = _line_content(response, cited_line.line, cited_line.markers)
+        if not content:
+            dropped[GroundingLineDropReason.NO_CONTENT] += 1
+            continue
+        for citation in cited_line.sources:
+            source = sources.setdefault(
+                citation.url,
+                _Source(url=citation.url, title=citation.title or citation.url),
+            )
+            source.contents[index] = content
+    return list(sources.values()), dropped
+
+
+def _line_content(
+    response: str,
+    line: _AnswerLine,
+    markers: Sequence[AnswerSpanCitation],
+) -> str:
+    """Return the line text without its list marker and citation markers.
+
+    Each marker is removed together with the whitespace before it, so
+    ``"- Claim 【4:0†source】."`` becomes ``"Claim."``.
+    """
+    list_marker = _LIST_MARKER_PATTERN.match(line.text)
+    cursor = line.start + (list_marker.end() if list_marker else 0)
+    parts: list[str] = []
+    for citation in sorted(markers, key=lambda item: item.start_index):
+        parts.append(response[cursor : citation.start_index].rstrip())
+        cursor = max(cursor, citation.end_index)
+    parts.append(response[cursor : line.end])
+    return "".join(parts).strip()