docsmind 0.6.0__tar.gz → 0.7.0__tar.gz

{docsmind-0.6.0 → docsmind-0.7.0}/CHANGELOG.md

@@ -2,6 +2,27 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.7.0] - 2026-11-12
+
+### Added
+
+- Change Analyzer can now look for a published changelog online when the documents themselves
+  contain none, using two new subagent-only tools:
+  - `web_search`: searches the web via Tavily, optionally limited to given domains. Only offered
+    when `DOCSMIND_WEB_SEARCH_API_KEY` is set.
+  - `fetch_url`: reads a public web page as text. Hardened for model-supplied URLs (http(s) on
+    ports 80/443 only, no credentials, no private/loopback addresses on any redirect hop,
+    bounded size and time).
+- Config: `web_search_api_key`, `web_timeout_s`, `web_fetch_max_bytes` (documented in
+  `.env.example`).
+- Implementation brief for the Change Analyzer (`docs/`) and unit tests for the web tools.
+
+### Changed
+
+- Change Analyzer prompt and tool list updated to use the web tools as a fallback, preferring
+  the publisher's own domain.
+- Rebuilt the bundled frontend UI assets.
+
 ## [0.6.0] - 2026-11-10
 
 ### Added

{docsmind-0.6.0 → docsmind-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: docsmind
-Version: 0.6.0
+Version: 0.7.0
 Summary: Chat with your documents. Version-aware retrieval, change analysis, and an editable table workspace, backed by Postgres/pgvector.
 Project-URL: Homepage, https://github.com/yauheniya-ai/docsmind
 Project-URL: Repository, https://github.com/yauheniya-ai/docsmind

{docsmind-0.6.0 → docsmind-0.7.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "docsmind"
-version = "0.6.0"
+version = "0.7.0"
 description = "Chat with your documents. Version-aware retrieval, change analysis, and an editable table workspace, backed by Postgres/pgvector."
 readme = "README.md"
 requires-python = ">=3.10"

{docsmind-0.6.0 → docsmind-0.7.0}/src/docsmind/agent/agent.py

@@ -75,6 +75,7 @@ class DocsmindAgent:
         self._memory = memory
         self._query_embedder = None
         self.tools = tools or build_registry()
+        self.main_tools = self.tools.for_executor()
         self.builder = ContextBuilder(config)
         self._background: set[asyncio.Task] = set()
 
@@ -187,7 +188,7 @@ class DocsmindAgent:
                     "with citations that are verified against what it actually read."
                 ),
                 "model": self.config.llm_model,
-                "tools": self.tools.info(),
+                "tools": self.main_tools.info(),
             },
             "stages": [
                 {
@@ -403,11 +404,11 @@ class DocsmindAgent:
             delegate=self._make_delegate(llm, catalog, sources, usage, emit),
             embedder=self.query_embedder,
         )
-        bundle = self.builder.executor(state, self.tools.specs())
+        bundle = self.builder.executor(state, self.main_tools.specs())
         result = await run_tool_loop(
             llm=llm,
             bundle=bundle,
-            registry=self.tools,
+            registry=self.main_tools,
             ctx=tool_ctx,
             usage=usage,
             emit=emit,
@@ -482,7 +483,7 @@ class DocsmindAgent:
                 delegate=None,  # no recursion
                 embedder=self.query_embedder,
             )
-            registry = self.tools.subset(spec.tools)
+            registry = self.tools.subset(spec.tools).configured(self.config)
             bundle = self.builder.subagent(spec, task, sub_catalog, registry.specs())
 
             emit(AgentEvent(type="subagent_start", data={"id": sub_id, "name": spec.name, "task": task}))

{docsmind-0.6.0 → docsmind-0.7.0}/src/docsmind/agent/prompts.py

@@ -22,12 +22,12 @@ You are the Mind, the assistant inside docsmind. You answer questions about the
 
 How to work
 - Documents are referred to by handle (D1, D2, ...); the register is listed below.
-- search: ranked keyword search; start here for most questions. Use distinctive words, not the whole question.
+- search: ranked hybrid search (keywords plus meaning); start here for most questions. Write the query in natural language, e.g. the question or a descriptive phrase.
 - grep: exact phrase or identifier (a clause number, a defined term). Use when search blurs exact wording.
 - read_pages: read the actual text of pages. Read around a promising hit before relying on a snippet.
 - get_outline: the document's table of contents with page numbers.
 - list_documents: only if you need page/word counts again.
-- delegate: hand a self-contained task to a subagent with a fresh context when the work is large or independent (comparing several documents or versions, reading many sections). It returns a short summary that keeps the same [S#] handles. Several delegate calls in one step run in parallel.
+- delegate: hand a self-contained task to a subagent with a fresh context when the work is large or independent (comparing several documents or versions, reading many sections). It returns a short summary that keeps the same [S#] handles; web pages it used appear as markdown links, which you keep as given. Several delegate calls in one step run in parallel.
 - Make independent tool calls in the same step. Prefer a few precise calls over many vague ones. Do not narrate before calling tools.
 - If the question is conversational or not about the documents, answer directly without tools.
 
@@ -73,7 +73,15 @@ Role: researcher. Gather the evidence needed for a focused question. Search, gre
 CHANGE_ANALYZER = f"""\
 {SUBAGENT_COMMON}
 
-Role: change analyzer. Compare two versions of the same document (or two related documents) and report what was added, removed or reworded, section by section, and why it matters. Use get_outline on both to align sections, then read_pages on matching sections. Cite both sides of every difference. If the versions are identical in the area asked about, say so."""
+Role: change analyzer. Compare two versions of the same document (or two related documents) and report what was added, removed or reworded, section by section, and why it matters.
+
+Step 1, always first: look for a changelog before comparing any text.
+- In the documents: search for "revision history", "summary of changes", "changelog", "what's new", "errata", "change log"; check get_outline for such a section (usually at the front or in an appendix), and read it with read_pages.
+- Online, only if web_search / fetch_url are among your tools and the documents give no usable changelog: search for the publisher's changes page for these exact versions (prefer the publisher's own domain), then fetch_url the best result. Follow a link from that page at most once or twice.
+- Treat a changelog as a map of where to look, not as proof. Confirm every change you report against the document text on both sides.
+- Cite changelog entries that come from a document with [S#]. For a web page, name it as a markdown link [title](url); never give a web page an [S#] handle. If you found no changelog, say so in one line and continue.
+
+Step 2: use get_outline on both to align sections, then read_pages on matching sections. Cite both sides of every difference. If the versions are identical in the area asked about, say so."""
 
 SUMMARIZER = """\
 You maintain the running summary of a conversation between a user and a document assistant. Merge the new turns into the existing summary.

{docsmind-0.6.0 → docsmind-0.7.0}/src/docsmind/agent/subagents/specs.py

@@ -35,10 +35,11 @@ CHANGE_ANALYZER = SubagentSpec(
     role="Version comparison",
     description=(
         "Compares two versions of a document (or two related documents) and reports what was "
-        "added, removed or reworded, section by section, with citations on both sides."
+        "added, removed or reworded, section by section, with citations on both sides. It first "
+        "looks for a changelog in the documents and, when web access is configured, online."
     ),
     prompt=prompts.CHANGE_ANALYZER,
-    tools=("search", "grep", "read_pages", "get_outline"),
+    tools=("search", "grep", "read_pages", "get_outline", "web_search", "fetch_url"),
 )
 
 SUBAGENTS: dict[str, SubagentSpec] = {s.name: s for s in (RESEARCHER, CHANGE_ANALYZER)}

{docsmind-0.6.0 → docsmind-0.7.0}/src/docsmind/agent/tools/__init__.py

@@ -6,13 +6,14 @@ from docsmind.agent.tools.base import Tool, ToolContext, ToolOutput, ToolRegistr
 from docsmind.agent.tools.delegate import DelegateTool
 from docsmind.agent.tools.documents import ListDocumentsTool, OutlineTool, ReadPagesTool
 from docsmind.agent.tools.search import GrepTool, SearchTool
+from docsmind.agent.tools.web import FetchUrlTool, WebSearchTool
 
 __all__ = ["Tool", "ToolContext", "ToolOutput", "ToolRegistry", "build_registry"]
 
 
 def build_registry() -> ToolRegistry:
-    """Every tool the main executor can call. Subagents get a subset (see
-    `agent/subagents/specs.py`) and never `delegate`."""
+    """Every tool. The main executor gets all but the `subagent_only` ones (the web tools);
+    subagents get the subset their spec lists (see `agent/subagents/specs.py`) and never `delegate`."""
     return ToolRegistry(
         [
             SearchTool(),
@@ -21,5 +22,7 @@ def build_registry() -> ToolRegistry:
             OutlineTool(),
             ListDocumentsTool(),
             DelegateTool(),
+            WebSearchTool(),
+            FetchUrlTool(),
         ]
     )

{docsmind-0.6.0 → docsmind-0.7.0}/src/docsmind/agent/tools/base.py

@@ -51,8 +51,15 @@ class Tool:
     name: ClassVar[str]
     description: ClassVar[str]  # written for the model
     summary: ClassVar[str] = ""  # one plain sentence for people (shown in the UI)
+    how_it_works: ClassVar[tuple[str, ...]] = ()  # quick-reference steps for people (UI only, never sent to the model)
+    subagent_only: ClassVar[bool] = False  # True: never offered to the main executor, only to subagents that list it
     Args: ClassVar[type[BaseModel]]
 
+    @classmethod
+    def available(cls, config: DocsmindConfig) -> bool:
+        """False when the tool needs configuration that is missing (it is then not offered to the model)."""
+        return True
+
     async def run(self, args: Any, ctx: ToolContext) -> ToolOutput:  # pragma: no cover
         raise NotImplementedError
 
@@ -103,6 +110,7 @@ class ToolRegistry:
                 {
                     "name": n,
                     "description": tool.summary,
+                    "how_it_works": list(tool.how_it_works),
                     "instructions": spec.description,
                     "parameters": spec.parameters,
                 }
@@ -112,6 +120,14 @@ class ToolRegistry:
     def subset(self, names: tuple[str, ...] | list[str]) -> ToolRegistry:
         return ToolRegistry([t for n, t in self._tools.items() if n in names])
 
+    def for_executor(self) -> ToolRegistry:
+        """The tools the main executor may call: everything except subagent-only tools."""
+        return ToolRegistry([t for t in self._tools.values() if not t.subagent_only])
+
+    def configured(self, config: DocsmindConfig) -> ToolRegistry:
+        """Drops tools whose configuration is missing, so the model never sees a tool it cannot use."""
+        return ToolRegistry([t for t in self._tools.values() if t.available(config)])
+
     def specs(self) -> list[ToolSpec]:
         return [t.spec() for t in self._tools.values()]
 

{docsmind-0.6.0 → docsmind-0.7.0}/src/docsmind/agent/tools/delegate.py

@@ -37,6 +37,11 @@ class DelegateTool(Tool):
         "short report with [S#] citations. Use for large or independent work; several delegate calls in "
         "one step run in parallel.\nSubagents:\n" + _ROSTER
     )
+    how_it_works = (
+        "The Mind picks a subagent and writes a self-contained task; the subagent never sees the conversation.",
+        "The subagent works in a fresh context with its own tools (search, grep, read_pages, get_outline) and cannot delegate further.",
+        "Only its short report comes back, with the same [S#] source handles. Several delegations in one step run in parallel.",
+    )
     Args = DelegateArgs
 
     async def run(self, args: DelegateArgs, ctx: ToolContext) -> ToolOutput:

{docsmind-0.6.0 → docsmind-0.7.0}/src/docsmind/agent/tools/documents.py

@@ -20,6 +20,10 @@ class ListDocumentsTool(Tool):
     name = "list_documents"
     summary = "Lists the documents in the register with their collection, version and size."
     description = "List the documents in the register with their handles, collection, version label, page and word counts."
+    how_it_works = (
+        "Reads the catalog built for this conversation (no database query).",
+        "Returns each document's handle (D1, D2, ...), title, collection, version, and page and word counts.",
+    )
     Args = ListDocumentsArgs
 
     async def run(self, args: ListDocumentsArgs, ctx: ToolContext) -> ToolOutput:
@@ -37,6 +41,11 @@ class OutlineTool(Tool):
         "The document's table of contents (from the PDF's own bookmarks) with page numbers. "
         "Use it to find which pages cover a topic, then read_pages."
     )
+    how_it_works = (
+        "Loads the document's sections, taken from the PDF's own bookmarks.",
+        "Lists them indented by depth, each with its page number (up to 200 entries).",
+        "If the PDF has no bookmarks, it says so and the model falls back to search.",
+    )
     Args = OutlineArgs
 
     async def run(self, args: OutlineArgs, ctx: ToolContext) -> ToolOutput:
@@ -74,6 +83,11 @@ class ReadPagesTool(Tool):
         "Read the full text of one or a few consecutive pages of a document. Each chunk comes with a "
         "source handle to cite."
     )
+    how_it_works = (
+        "Checks the page range: at most 4 pages per call, and within the document.",
+        "Loads every text chunk that overlaps those pages.",
+        "Returns the full text, each chunk with a source handle to cite.",
+    )
     Args = ReadPagesArgs
 
     async def run(self, args: ReadPagesArgs, ctx: ToolContext) -> ToolOutput:

{docsmind-0.6.0 → docsmind-0.7.0}/src/docsmind/agent/tools/search.py

@@ -24,7 +24,14 @@ def format_passage(doc: CatalogDoc, handle: str, page_start: int | None, page_en
 
 
 class SearchArgs(BaseModel):
-    query: str = Field(min_length=2, description="Distinctive words to look for (not a full sentence).")
+    query: str = Field(
+        min_length=2,
+        max_length=1000,
+        description=(
+            "Natural-language query: a question or a descriptive phrase. "
+            "Use grep instead for exact wording, identifiers or clause numbers."
+        ),
+    )
     documents: list[str] | None = Field(default=None, description=_DOCUMENTS_HELP)
     limit: int = Field(default=6, ge=1, le=10, description="Maximum passages to return.")
 
@@ -32,14 +39,19 @@ class SearchArgs(BaseModel):
 class SearchTool(Tool):
     name = "search"
     summary = (
-        "Searches the documents by keywords, and by meaning once they have been embedded. "
-        "Returns the best matching passages to cite."
+        "Hybrid search over the documents: stemmed keyword matching plus semantic (embedding) similarity, "
+        "fused by reciprocal rank fusion. Returns the best matching passages to cite."
     )
     description = (
-        "Search the documents' text (and figure captions, if images were captioned). Matches by "
-        "stemmed keywords, plus by meaning when the documents have been embedded, so paraphrases can "
-        "match too. Returns the best passages, each with a source handle like [S1] to cite; use "
-        "read_pages on a hit to see the surrounding text."
+        "Find passages by topic or meaning in the documents' text and figure captions. Takes a "
+        "natural-language query; matches keywords and paraphrases, ranked best first. Returns passages "
+        "with a source handle like [S1] to cite; use read_pages to see the surrounding text. "
+    )
+    how_it_works = (
+        "Keyword search: the query is stemmed and matched with Postgres full-text search.",
+        "Semantic search: the query is embedded and the nearest passages are found by cosine similarity (skipped if the documents are not embedded).",
+        "Reciprocal rank fusion: each list scores a passage 1 / (60 + rank) and the scores are added, so passages ranked well by both come first.",
+        "The top passages come back with source handles to cite.",
     )
     Args = SearchArgs
 
@@ -102,6 +114,11 @@ class GrepTool(Tool):
         "Find an exact phrase, clause number or identifier (case-insensitive, literal match, no stemming). "
         "Returns the pages containing it, each with a source handle to cite."
     )
+    how_it_works = (
+        "Escapes the text and runs a case-insensitive substring match (ILIKE) over every page of the chosen documents.",
+        "No stemming and no ranking: only literal matches, in document and page order.",
+        "Returns a snippet around the first hit on each page, with the number of occurrences and a source handle.",
+    )
     Args = GrepArgs
 
     async def run(self, args: GrepArgs, ctx: ToolContext) -> ToolOutput:

docsmind-0.7.0/src/docsmind/agent/tools/web.py

@@ -0,0 +1,273 @@
+"""Web tools, for the Change Analyzer only: find and read a published changelog.
+
+Everything a web page says is untrusted DATA, like document text. The fetcher
+is hardened because the URL comes from a model: http(s) on ports 80/443 only,
+no credentials in the URL, no private/loopback/link-local addresses (checked on
+every redirect hop), bounded size and time.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import ipaddress
+import re
+import socket
+from html.parser import HTMLParser
+from urllib.parse import urljoin, urlsplit
+
+import httpx
+from pydantic import BaseModel, Field
+
+from docsmind.agent.tools.base import Tool, ToolContext, ToolOutput
+from docsmind.config import DocsmindConfig
+from docsmind.domain.errors import ToolInputError
+
+TAVILY_URL = "https://api.tavily.com/search"
+PAGE_CHARS = 10_000  # per fetch_url call; below agent_tool_result_max_chars so it is never cut again
+MAX_REDIRECTS = 4
+ALLOWED_PORTS = {None, 80, 443}
+TEXT_TYPES = ("text/html", "application/xhtml", "text/plain", "text/markdown")
+USER_AGENT = "docsmind/1.0 (document assistant; reads public pages)"
+
+
+# -- web_search ------------------------------------------------------------------------
+
+
+class WebSearchArgs(BaseModel):
+    query: str = Field(min_length=3, max_length=400, description="What to look for on the web.")
+    domains: list[str] | None = Field(
+        default=None,
+        max_length=5,
+        description="Only search these sites, e.g. the publisher's domain. Omit to search the whole web.",
+    )
+    limit: int = Field(default=5, ge=1, le=8, description="Maximum results to return.")
+
+
+class WebSearchTool(Tool):
+    name = "web_search"
+    subagent_only = True
+    summary = (
+        "Searches the public web, mainly to find a published changelog or release notes for a document. "
+        "Needs a Tavily API key (DOCSMIND_WEB_SEARCH_API_KEY)."
+    )
+    description = (
+        "Search the public web. Returns the title, URL and an excerpt of each result. Use it to find "
+        "an official changelog, revision history or release notes for a document; prefer the "
+        "publisher's own site. Read a result in full with fetch_url."
+    )
+    how_it_works = (
+        "Sends the query (optionally limited to given domains) to the Tavily search API.",
+        "Returns each result's title, URL and excerpt. The model reads promising pages with fetch_url.",
+        "Only offered when DOCSMIND_WEB_SEARCH_API_KEY is set.",
+    )
+    Args = WebSearchArgs
+
+    @classmethod
+    def available(cls, config: DocsmindConfig) -> bool:
+        return bool(config.web_search_api_key)
+
+    async def run(self, args: WebSearchArgs, ctx: ToolContext) -> ToolOutput:
+        key = ctx.config.web_search_api_key
+        if not key:
+            raise ToolInputError("Web search is not configured. Work from the documents only.")
+        payload: dict = {
+            "query": args.query,
+            "max_results": args.limit,
+            "search_depth": "basic",
+            "include_answer": False,
+        }
+        if args.domains:
+            payload["include_domains"] = args.domains
+        try:
+            async with httpx.AsyncClient(timeout=ctx.config.web_timeout_s) as client:
+                response = await client.post(TAVILY_URL, json=payload, headers={"Authorization": f"Bearer {key}"})
+        except httpx.HTTPError as exc:
+            raise ToolInputError("Web search is unreachable right now. Work from the documents only.") from exc
+        if response.status_code in (401, 403):
+            raise ToolInputError("Web search rejected its API key. Work from the documents only.")
+        if response.status_code >= 400:
+            raise ToolInputError(f"Web search failed (HTTP {response.status_code}). Try different words or skip it.")
+
+        results = response.json().get("results") or []
+        if not results:
+            return ToolOutput(
+                f"No web results for '{args.query}'. Try other words or a different site.", summary="no results"
+            )
+        lines = []
+        for i, result in enumerate(results[: args.limit], start=1):
+            excerpt = re.sub(r"\s+", " ", result.get("content") or "").strip()[:500]
+            lines.append(f"{i}. {result.get('title') or '(untitled)'}\n   {result.get('url')}\n   {excerpt}")
+        return ToolOutput("\n\n".join(lines), summary=f"{len(lines)} results")
+
+
+# -- fetch_url -------------------------------------------------------------------------
+
+
+class FetchUrlArgs(BaseModel):
+    url: str = Field(min_length=8, max_length=2000, description="Full http(s) URL of a public web page.")
+    offset: int = Field(default=0, ge=0, description="Character to start from, to continue a long page.")
+
+
+class FetchUrlTool(Tool):
+    name = "fetch_url"
+    subagent_only = True
+    summary = "Reads the text of one public web page, with its links, for example a changelog or release notes."
+    description = (
+        "Fetch a public web page (HTML or plain text; not PDFs) and return its text, with link targets "
+        "shown as 'text (url)'. Long pages come in parts: pass the offset given at the end to continue. "
+        "Use it on a changelog, revision history or release-notes page."
+    )
+    how_it_works = (
+        "Accepts only public http(s) URLs: private and local addresses are refused, also after redirects.",
+        "Downloads the page (size and time limited) and converts the HTML to plain text, keeping link URLs.",
+        "Returns the text in parts of about 10,000 characters; the model asks for the next part with an offset.",
+    )
+    Args = FetchUrlArgs
+
+    async def run(self, args: FetchUrlArgs, ctx: ToolContext) -> ToolOutput:
+        final_url, content_type, body = await _fetch(args.url, ctx.config)
+        if "html" in content_type or "xml" in content_type:
+            title, text = _html_to_text(body, final_url)
+        else:
+            title, text = "", body.strip()
+        if not text:
+            return ToolOutput(f"{final_url} has no readable text.", summary="no text")
+
+        total = len(text)
+        if args.offset >= total:
+            raise ToolInputError(f"offset is past the end of the page ({total} characters).")
+        end = min(total, args.offset + PAGE_CHARS)
+        header = f"Web page: {final_url}" + (f"\nTitle: {title}" if title else "")
+        footer = (
+            f"\n[Showing characters {args.offset}-{end} of {total}. Call again with offset={end} for more.]"
+            if end < total
+            else ""
+        )
+        host = urlsplit(final_url).hostname or final_url
+        return ToolOutput(f"{header}\n\n{text[args.offset:end]}{footer}", summary=f"{host} ({total} chars)")
+
+
+async def _assert_public(host: str) -> None:
+    try:
+        infos = await asyncio.get_running_loop().getaddrinfo(host, None, type=socket.SOCK_STREAM)
+    except socket.gaierror as exc:
+        raise ToolInputError(f"Cannot resolve {host}.") from exc
+    for info in infos:
+        ip = ipaddress.ip_address(info[4][0].split("%")[0])
+        if ip.version == 6 and ip.ipv4_mapped is not None:
+            ip = ip.ipv4_mapped
+        if not ip.is_global:
+            raise ToolInputError("That address is on a private or local network and cannot be fetched.")
+
+
+def _check_url(url: str) -> str:
+    parts = urlsplit(url.strip())
+    if parts.scheme not in ("http", "https") or not parts.hostname:
+        raise ToolInputError("Only full http(s) URLs can be fetched.")
+    if parts.username or parts.password:
+        raise ToolInputError("URLs with credentials are not allowed.")
+    try:
+        port = parts.port
+    except ValueError as exc:
+        raise ToolInputError("The URL has an invalid port.") from exc
+    if port not in ALLOWED_PORTS:
+        raise ToolInputError("Only the standard web ports (80, 443) are allowed.")
+    return parts.geturl()
+
+
+async def _fetch(url: str, config: DocsmindConfig) -> tuple[str, str, str]:
+    """-> (final url, content type, decoded body). Redirects are followed by hand so every hop is checked."""
+    url = _check_url(url)
+    headers = {"User-Agent": USER_AGENT, "Accept": "text/html,text/plain;q=0.9,*/*;q=0.1"}
+    async with httpx.AsyncClient(timeout=config.web_timeout_s, follow_redirects=False, headers=headers) as client:
+        for _ in range(MAX_REDIRECTS + 1):
+            await _assert_public(urlsplit(url).hostname or "")
+            try:
+                async with client.stream("GET", url) as response:
+                    if response.is_redirect:
+                        location = response.headers.get("location")
+                        if not location:
+                            raise ToolInputError("The server redirected without saying where.")
+                        url = _check_url(urljoin(url, location))
+                        continue
+                    if response.status_code >= 400:
+                        raise ToolInputError(f"The server answered HTTP {response.status_code} for {url}.")
+                    content_type = response.headers.get("content-type", "").split(";")[0].strip().lower()
+                    if content_type and not content_type.startswith(TEXT_TYPES):
+                        raise ToolInputError(
+                            f"{url} is {content_type}, which cannot be read here. Only HTML or plain text pages work."
+                        )
+                    data = bytearray()
+                    async for chunk in response.aiter_bytes():
+                        data += chunk
+                        if len(data) > config.web_fetch_max_bytes:
+                            break
+                    encoding = response.encoding or "utf-8"
+                    try:
+                        return url, content_type or "text/html", bytes(data).decode(encoding, errors="replace")
+                    except LookupError:
+                        return url, content_type or "text/html", bytes(data).decode("utf-8", errors="replace")
+            except httpx.HTTPError as exc:
+                raise ToolInputError(f"Could not fetch {url}: {type(exc).__name__}.") from exc
+    raise ToolInputError("Too many redirects.")
+
+
+_BLOCK_TAGS = {
+    "p", "div", "br", "li", "tr", "section", "article", "header", "footer", "blockquote", "pre",
+    "ul", "ol", "table", "h1", "h2", "h3", "h4", "h5", "h6", "dt", "dd", "hr",
+}  # fmt: skip
+_SKIP_TAGS = {"script", "style", "noscript", "svg", "template"}
+
+
+class _TextExtractor(HTMLParser):
+    """Visible text, one block per line, with links written as 'text (url)'."""
+
+    def __init__(self, base_url: str) -> None:
+        super().__init__(convert_charrefs=True)
+        self.base_url = base_url
+        self.title = ""
+        self.parts: list[str] = []
+        self._skip = 0
+        self._in_title = False
+        self._links: list[str | None] = []
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        if tag in _SKIP_TAGS:
+            self._skip += 1
+        elif tag == "title":
+            self._in_title = True
+        elif tag == "a":
+            href = dict(attrs).get("href") or ""
+            target = urljoin(self.base_url, href) if href and not href.startswith(("#", "javascript:", "mailto:")) else None
+            self._links.append(target if target and target.startswith(("http://", "https://")) else None)
+        if tag in _BLOCK_TAGS:
+            self.parts.append("\n")
+
+    def handle_endtag(self, tag: str) -> None:
+        if tag in _SKIP_TAGS and self._skip:
+            self._skip -= 1
+        elif tag == "title":
+            self._in_title = False
+        elif tag == "a" and self._links:
+            target = self._links.pop()
+            if target:
+                self.parts.append(f" ({target})")
+        if tag in _BLOCK_TAGS:
+            self.parts.append("\n")
+
+    def handle_data(self, data: str) -> None:
+        if self._in_title:
+            self.title += data
+        elif not self._skip:
+            self.parts.append(data)
+
+
+def _html_to_text(html: str, base_url: str) -> tuple[str, str]:
+    parser = _TextExtractor(base_url)
+    parser.feed(html)
+    parser.close()
+    text = "".join(parser.parts)
+    text = re.sub(r"[ \t\r\f\v]+", " ", text)
+    text = re.sub(r" ?\n ?", "\n", text)
+    text = re.sub(r"\n{2,}", "\n", text).strip()
+    return re.sub(r"\s+", " ", parser.title).strip(), text

{docsmind-0.6.0 → docsmind-0.7.0}/src/docsmind/config.py

@@ -62,6 +62,13 @@ class DocsmindConfig(BaseSettings):
     agent_recent_turns: int = 4  # user/assistant pairs always kept verbatim
     agent_memory_enabled: bool = True
 
+    # --- web access (Change Analyzer only) ---
+    # Tavily API key (https://tavily.com). Unset = the `web_search` tool is not offered; `fetch_url`
+    # (read one public web page) needs no key and stays available.
+    web_search_api_key: str | None = None
+    web_timeout_s: float = 15.0  # per web request
+    web_fetch_max_bytes: int = 2_000_000  # page bytes read before giving up
+
     # --- ingestion ---
     chunk_size_tokens: int = 700
     chunk_overlap_tokens: int = 100

{docsmind-0.6.0 → docsmind-0.7.0}/src/docsmind/retrieval/keyword.py

@@ -5,7 +5,9 @@ from __future__ import annotations
 import re
 
 _TOKEN_RE = re.compile(r"[^\W_]+", re.UNICODE)
-MAX_QUERY_TERMS = 12
+# Safety bound only (keeps the tsquery a sane size for pathological input);
+# normal questions never get near it, and the vector side embeds the full text.
+MAX_QUERY_TERMS = 64
 
 
 def query_terms(text: str) -> list[str]: