hypermind 2.1.0__tar.gz → 2.1.2__tar.gz

{hypermind-2.1.0 → hypermind-2.1.2}/.github/workflows/release.yml

@@ -5,14 +5,6 @@ on:
     tags:
       - 'v*'
   workflow_dispatch:
-    inputs:
-      version:
-        description: >-
-          Version tag to publish (e.g. v2.1.0). Only used when this workflow
-          is run manually instead of by a tag push — TEMPORARY escape hatch,
-          revert once the tag-push path is available again.
-        required: true
-        type: string
 
 permissions:
   contents: read
@@ -168,7 +160,7 @@ jobs:
     name: publish → PyPI
     runs-on: ubuntu-latest
     needs: build
-    if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch'
+    if: startsWith(github.ref, 'refs/tags/v')
     environment:
       name: pypi
       url: https://pypi.org/p/hypermind
@@ -241,22 +233,12 @@ jobs:
     name: docker — hub.docker.com/openworkbench/hypermind
     runs-on: ubuntu-latest
     needs: build
-    if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch'
+    if: startsWith(github.ref, 'refs/tags/v')
     permissions:
       contents: read
     steps:
       - uses: actions/checkout@v4
 
-      - name: resolve release tag
-        id: reltag
-        run: |
-          set -euo pipefail
-          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
-            echo "tag=${{ github.event.inputs.version }}" >> "$GITHUB_OUTPUT"
-          else
-            echo "tag=${{ github.ref_name }}" >> "$GITHUB_OUTPUT"
-          fi
-
       - name: set up QEMU (multi-arch)
         uses: docker/setup-qemu-action@v3
 
@@ -283,7 +265,7 @@ jobs:
           platforms: linux/amd64,linux/arm64
           tags: |
             openworkbench/hypermind:latest
-            openworkbench/hypermind:${{ steps.reltag.outputs.tag }}
+            openworkbench/hypermind:${{ github.ref_name }}
             openworkbench/hypermind:${{ steps.sha.outputs.short }}
           cache-from: type=gha
           cache-to: type=gha,mode=max

{hypermind-2.1.0 → hypermind-2.1.2}/CHANGELOG.md

@@ -14,6 +14,26 @@ changes only on major-version bumps.
 
 ## [Unreleased]
 
+## [2.1.2] — 2026-07-27
+
+### Added
+- **`hmctl serve-endpoint` LLM responder auto-wiring**: auto-detects `OPENAI_API_KEY` / `OPENAI_BASE_URL` / `HYPERMIND_LLM_MODEL` from env and wires `OpenAIResponder` at startup — all three models (`hm-network`, `hm-corpus`, `hm-moa`) now generate real LLM rationale when a key is present. Falls back to reputation-only with a warning when no key is set.
+- **`--system-prompt` flag on `serve-endpoint`**: default scopes every panel member to swarm-verified knowledge and prefixes answers with `"HyperMind says — "`. Override with `--system-prompt "..."` per invocation.
+- **`"HyperMind says — "` response prefix**: all `/v1/chat/completions` responses (streaming and non-streaming) are now prefixed, making it clear answers come from the HyperMind swarm.
+
+### Fixed
+- Docker `CMD`: `hmctl mcp serve` now explicitly passes `--write --log-level info` — connections and tool calls are visible in `docker logs` output.
+
+## [2.1.1] — 2026-07-27
+
+### Added
+- **Web search result capture** (#973): agents now fetch every search result URL (previously capped at 3), convert HTML to clean markdown via `trafilatura` (strips nav/ads/footers), and store in Relata `/memory/remember` under `{agent_id}/web-search/{url_hash}` with full metadata. Each page is linked to its originating claim via `associate("source_for")`. Future rounds can recall full page content via `/memory/recall` instead of snippets only. `trafilatura>=1.6` added to the `[ingest]` optional extra; falls back to bs4 then skips gracefully.
+- **`hmctl status` endpoint panel**: a dedicated always-visible panel shows HyperMind's OpenAI-compatible endpoint URL, connection snippet, all models with descriptions, and all routes — turns green when live.
+- **`GET /v1/models` spec compliance**: response now includes `created` (Unix timestamp) alongside `id`, `object`, `owned_by`, and `description` — fully OpenAI-spec-compliant.
+
+### Fixed
+- Docker: `hmctl mcp serve` env file no longer overrides compose-set env vars (e.g. `RELATA_URL`) — process env takes priority over `/data/env` file values.
+
 ## [2.1.0] — 2026-07-27
 
 ### Added

{hypermind-2.1.0 → hypermind-2.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hypermind
-Version: 2.1.0
+Version: 2.1.2
 Summary: Federated agent intelligence — reference SDK for the HyperMind SCITT contested-claims profile
 Project-URL: Homepage, https://github.com/ZySec-AI/hypermind
 Project-URL: Documentation, https://github.com/ZySec-AI/hypermind/tree/main/docs
@@ -55,6 +55,7 @@ Provides-Extra: ingest
 Requires-Dist: beautifulsoup4>=4.12; extra == 'ingest'
 Requires-Dist: pypdf>=4.0; extra == 'ingest'
 Requires-Dist: python-docx>=1.0; extra == 'ingest'
+Requires-Dist: trafilatura>=1.6; extra == 'ingest'
 Provides-Extra: libp2p
 Provides-Extra: mcp
 Requires-Dist: mcp>=1.0; extra == 'mcp'

{hypermind-2.1.0 → hypermind-2.1.2}/deploy/Dockerfile

@@ -60,4 +60,4 @@ EXPOSE 8484 8766
 #   OPENAI_BASE_URL=http://...     # local model (Ollama etc.)
 #   HYPERMIND_LLM_MODEL=gpt-4o
 #   RELATA_URL=http://relata:9090  # optional durable ledger
-CMD ["hmctl", "mcp", "serve"]
+CMD ["hmctl", "mcp", "serve", "--write", "--log-level", "info"]

{hypermind-2.1.0 → hypermind-2.1.2}/deploy/Dockerfile.mcp

@@ -16,4 +16,4 @@ VOLUME ["/config"]
 # TCP MCP (multi-user JSON-RPC), SSE dashboard
 EXPOSE 8484 8766
 
-CMD ["hmctl", "mcp", "serve", "--config", "/config/mcp.toml"]
+CMD ["hmctl", "mcp", "serve", "--config", "/config/mcp.toml", "--write", "--log-level", "info"]

{hypermind-2.1.0 → hypermind-2.1.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "hypermind"
-version = "2.1.0"
+version = "2.1.2"
 description = "Federated agent intelligence — reference SDK for the HyperMind SCITT contested-claims profile"
 readme = "README.md"
 requires-python = ">=3.12,<3.15"
@@ -83,6 +83,8 @@ ingest = [
     "pypdf>=4.0",
     "python-docx>=1.0",
     "beautifulsoup4>=4.12",
+    # #973: web page fetch + markdown conversion for search result capture.
+    "trafilatura>=1.6",
 ]
 vector = [
     # #854: local (in-process, no external embedding service) semantic

{hypermind-2.1.0 → hypermind-2.1.2}/src/hmctl/cli.py

@@ -516,7 +516,8 @@ def status_cmd(as_json: bool) -> None:
 
     otel_ep = _os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "")
     db_path = _os.environ.get("HYPERMIND_DB", "")
-    openai_key = bool(_os.environ.get("OPENAI_API_KEY"))
+    openai_key_raw = _os.environ.get("OPENAI_API_KEY", "")
+    openai_key = bool(openai_key_raw)
     anthropic_key = bool(_os.environ.get("ANTHROPIC_API_KEY"))
     base_url = _os.environ.get("OPENAI_BASE_URL", "")
     llm_model = _os.environ.get("HYPERMIND_LLM_MODEL", "")
@@ -541,6 +542,41 @@ def status_cmd(as_json: bool) -> None:
             "  # Ollama (no key): OPENAI_BASE_URL=http://localhost:11434/v1"
         )
 
+    # HyperMind's own OpenAI-compatible endpoint (hmctl serve-endpoint)
+    _hm_api_host = _os.environ.get("HYPERMIND_API_HOST", "127.0.0.1")
+    _hm_api_port = int(_os.environ.get("HYPERMIND_API_PORT", "8899"))
+    _hm_api_url = f"http://{_hm_api_host}:{_hm_api_port}/v1"
+    # probe if it's actually running
+    try:
+        import socket as _sock
+
+        _s = _sock.socket()
+        _s.settimeout(0.5)
+        _s.connect((_hm_api_host, _hm_api_port))
+        _s.close()
+        _hm_api_running = True
+    except Exception:
+        _hm_api_running = False
+    if _hm_api_running:
+        _oai_ok = True
+        _oai_detail = (
+            f"[bold]{_hm_api_url}[/bold]  (live)\n"
+            f"  POST /v1/chat/completions  GET /v1/models  GET /healthz\n"
+            f"  models: hm-network, hm-corpus, hm-moa\n"
+            f"  client = OpenAI(base_url='{_hm_api_url}', api_key='hm')"
+        )
+        _oai_fix = ""
+    else:
+        _oai_ok = False
+        _oai_detail = (
+            f"{_hm_api_url}  (not running)\n"
+            f"  POST /v1/chat/completions backed by reputation-weighted swarm\n"
+            f"  models: hm-network, hm-corpus, hm-moa"
+        )
+        _oai_fix = (
+            "hmctl serve-endpoint --host 0.0.0.0 --port 8899  # pip install 'hypermind[api]' first"
+        )
+
     rows = [
         # (feature, ok, detail, fix)
         (
@@ -552,6 +588,7 @@ def status_cmd(as_json: bool) -> None:
             f"touch {_HM_HOME}/env  # one KEY=VALUE per line",
         ),
         ("llm", llm_ok, llm_note, llm_fix),
+        ("openai_endpoint", _oai_ok, _oai_detail, _oai_fix),
         (
             "relata",
             *(
@@ -681,6 +718,49 @@ def status_cmd(as_json: bool) -> None:
 
     console.print(table)
 
+    # OpenAI-compatible endpoint connection panel — always shown
+    from rich.panel import Panel
+
+    _ep_models = (
+        "  [bold]hm-network[/bold]  Live reputation-weighted swarm consult — best for open questions\n"
+        "  [bold]hm-corpus [/bold]  Query learned knowledge from past research runs (no LLM calls)\n"
+        "  [bold]hm-moa    [/bold]  Mixture-of-Agents: proposer fan-out → aggregation (highest quality)"
+    )
+    if _hm_api_running:
+        _ep_status = "[green]● LIVE[/green]"
+        _ep_snippet = (
+            f"[bold]base_url[/bold] = [cyan]{_hm_api_url}[/cyan]   [bold]api_key[/bold] = [dim]'hm'[/dim] [dim](any string)[/dim]\n\n"
+            f"[dim]Python:[/dim]  from openai import OpenAI\n"
+            f"          client = OpenAI(base_url=[cyan]'{_hm_api_url}'[/cyan], api_key=[cyan]'hm'[/cyan])\n"
+            f"          client.chat.completions.create(model=[cyan]'hm-network'[/cyan], messages=[...])\n\n"
+            f"[dim]curl:  [/dim]  curl {_hm_api_url}/chat/completions \\\n"
+            f"            -H 'Content-Type: application/json' \\\n"
+            f'            -d \'{{"model":"hm-network","messages":[{{"role":"user","content":"your question"}}]}}\'\n\n'
+            f"[bold]Models:[/bold]\n{_ep_models}\n\n"
+            f"[dim]Routes:[/dim]  POST /v1/chat/completions · GET /v1/models · GET /healthz\n"
+            f"          POST /v1/hm/publish · POST /v1/hm/ask · GET /v1/hm/agents · GET /v1/hm/reputation"
+        )
+    else:
+        _ep_status = "[yellow]● NOT RUNNING[/yellow]"
+        _ep_snippet = (
+            f"[bold]base_url[/bold] = [cyan]{_hm_api_url}[/cyan]   [bold]api_key[/bold] = [dim]'hm'[/dim] [dim](any string)[/dim]\n\n"
+            f"[dim]Python:[/dim]  from openai import OpenAI\n"
+            f"          client = OpenAI(base_url=[cyan]'{_hm_api_url}'[/cyan], api_key=[cyan]'hm'[/cyan])\n"
+            f"          client.chat.completions.create(model=[cyan]'hm-network'[/cyan], messages=[...])\n\n"
+            f"[bold]Models:[/bold]\n{_ep_models}\n\n"
+            f"[dim]Routes:[/dim]  POST /v1/chat/completions · GET /v1/models · GET /healthz\n"
+            f"          POST /v1/hm/publish · POST /v1/hm/ask · GET /v1/hm/agents · GET /v1/hm/reputation\n\n"
+            f"[yellow]▶ Start:[/yellow]  hmctl serve-endpoint --host 0.0.0.0 --port {_hm_api_port}"
+        )
+    console.print(
+        Panel(
+            _ep_snippet,
+            title=f"[bold]HyperMind OpenAI-compatible endpoint[/bold]  {_ep_status}",
+            border_style="green" if _hm_api_running else "yellow",
+            padding=(1, 2),
+        )
+    )
+
     # MCP servers panel
     if _mcp_servers:
         from rich.table import Table as _Table
@@ -2314,12 +2394,22 @@ def claims_list_cmd(topic: str | None, limit: int, room: str) -> None:
     console.print(table)
 
 
+_DEFAULT_SYSTEM_PROMPT = (
+    "You are HyperMind — a federated agent intelligence network. "
+    "Answer only from what the HyperMind swarm knows. "
+    "Prefix every answer with 'HyperMind says — ' and stay within swarm-verified knowledge."
+)
+
+
 @main.command("serve-endpoint")
 @click.option("--host", default="127.0.0.1")
 @click.option("--port", type=int, default=8899)
 @click.option("--swarm", type=int, default=3)
-def serve_endpoint_cmd(host: str, port: int, swarm: int) -> None:
+@click.option("--system-prompt", default=None, help="Override default HyperMind system prompt.")
+def serve_endpoint_cmd(host: str, port: int, swarm: int, system_prompt: str | None) -> None:
     """Serve an OpenAI-compatible /v1/chat/completions backed by the HM network."""
+    import os as _os_ep
+
     try:
         import uvicorn
     except ImportError:
@@ -2329,7 +2419,36 @@ def serve_endpoint_cmd(host: str, port: int, swarm: int) -> None:
     from hypermind.integrations import HyperMindClient
     from hypermind.integrations.openai_endpoint import create_app
 
-    client = HyperMindClient.sandbox(swarm=swarm)
+    _prompt = system_prompt or _DEFAULT_SYSTEM_PROMPT
+
+    # Auto-wire OpenAI-compatible responder from env
+    responder = None
+    _api_key = _os_ep.environ.get("OPENAI_API_KEY", "")
+    _base_url = _os_ep.environ.get("OPENAI_BASE_URL", "")
+    _model = _os_ep.environ.get("HYPERMIND_LLM_MODEL", "gpt-4o-mini")
+    if _api_key:
+        try:
+            from openai import AsyncOpenAI
+
+            from hypermind.responders import OpenAIResponder
+
+            responder = OpenAIResponder(
+                client=AsyncOpenAI(api_key=_api_key, base_url=_base_url or None),
+                model=_model,
+                system_prompt=_prompt,
+            )
+            console.print(
+                f"[green]✓[/] LLM responder: [bold]{_model}[/]"
+                + (f"  via {_base_url}" if _base_url else "")
+            )
+        except ImportError:
+            console.print("[yellow]~ openai package not installed — reputation-only mode[/]")
+    else:
+        console.print(
+            "[yellow]~ No OPENAI_API_KEY — reputation-only (set key for LLM rationale)[/]"
+        )
+
+    client = HyperMindClient.sandbox(swarm=swarm, responder=responder)
     console.print(
         f"[bold]HyperMind endpoint[/] on http://{host}:{port}/v1  "
         f"(model: hm-network) — point any OpenAI-compatible base_url here"
@@ -3085,7 +3204,7 @@ def mcp_serve_cmd(
                         continue
                     _hek, _, _hev = _heline.partition("=")
                     _hek = _hek.strip()
-                    if _hek:
+                    if _hek and _hek not in _os_serve.environ:
                         _os_serve.environ[_hek] = _hev.strip()
 
         async def _resolve_tcp_port(host: str, port: int) -> int | None:

{hypermind-2.1.0 → hypermind-2.1.2}/src/hypermind/integrations/openai_endpoint.py

@@ -120,8 +120,12 @@ def _hypermind_ext(ans: NetworkAnswer) -> dict[str, Any]:
     }
 
 
+_HM_PREFIX = "HyperMind says — "
+
+
 def _completion_payload(model: str, ans: NetworkAnswer) -> dict[str, Any]:
-    return completion_payload(model, ans.text, extra={"hypermind": _hypermind_ext(ans)})
+    text = ans.text if ans.text.startswith(_HM_PREFIX) else _HM_PREFIX + ans.text
+    return completion_payload(model, text, extra={"hypermind": _hypermind_ext(ans)})
 
 
 def _sse_chunks(model: str, ans: NetworkAnswer) -> list[bytes]:
@@ -138,7 +142,8 @@ def _sse_chunks(model: str, ans: NetworkAnswer) -> list[bytes]:
             extra={"hypermind": _hypermind_ext(ans)},
         )
     ]
-    for piece in chunk_text(ans.text):
+    text = ans.text if ans.text.startswith(_HM_PREFIX) else _HM_PREFIX + ans.text
+    for piece in chunk_text(text):
         frames.append(
             sse_frame(model, {"content": piece}, None, completion_id=completion_id, created=created)
         )
@@ -477,19 +482,33 @@ async def _handle_chat_completions(
     await _send_json(send, 200, _completion_payload(model, ans))
 
 
+_HM_MODELS = [
+    {
+        "id": "hm-network",
+        "object": "model",
+        "created": 1753920000,  # 2025-07-31 — HyperMind 1.0 GA
+        "owned_by": "hypermind",
+        "description": "Live reputation-weighted swarm consult. Best for open questions — answers are provenance-signed and contestable.",
+    },
+    {
+        "id": "hm-corpus",
+        "object": "model",
+        "created": 1753920000,
+        "owned_by": "hypermind",
+        "description": "Query learned knowledge from past research runs. No live LLM calls — fast, deterministic, grounded in prior sim output.",
+    },
+    {
+        "id": "hm-moa",
+        "object": "model",
+        "created": 1753920000,
+        "owned_by": "hypermind",
+        "description": "Mixture-of-Agents: proposer fan-out followed by reputation-weighted aggregation. Highest quality, requires a learned topic.",
+    },
+]
+
+
 async def _handle_models(send: Send) -> None:
-    await _send_json(
-        send,
-        200,
-        {
-            "object": "list",
-            "data": [
-                {"id": "hm-network", "object": "model", "owned_by": "hypermind"},
-                {"id": "hm-corpus", "object": "model", "owned_by": "hypermind"},
-                {"id": _MOA_MODEL, "object": "model", "owned_by": "hypermind"},
-            ],
-        },
-    )
+    await _send_json(send, 200, {"object": "list", "data": _HM_MODELS})
 
 
 # --------------------------------------------------------------------------

{hypermind-2.1.0 → hypermind-2.1.2}/src/hypermind/moa/codeexec.py

@@ -82,8 +82,15 @@ def _preexec_limits(cpu_seconds: int, address_space_bytes: int) -> Any:
         return None
 
     def _limit() -> None:
-        resource.setrlimit(resource.RLIMIT_CPU, (cpu_seconds, cpu_seconds))
-        resource.setrlimit(resource.RLIMIT_AS, (address_space_bytes, address_space_bytes))
+        # RLIMIT_CPU is broadly supported; RLIMIT_AS may raise on some kernels/macOS.
+        try:
+            resource.setrlimit(resource.RLIMIT_CPU, (cpu_seconds, cpu_seconds))
+        except (OSError, ValueError):
+            pass
+        try:
+            resource.setrlimit(resource.RLIMIT_AS, (address_space_bytes, address_space_bytes))
+        except (OSError, ValueError):
+            pass
 
     return _limit
 

{hypermind-2.1.0 → hypermind-2.1.2}/src/hypermind/orgs/sim.py

@@ -4836,20 +4836,22 @@ class OrgSim:
         # Top-3 primary-source URLs ride ON the signed claim so the ledger is
         # self-attributing — verifiable without access to the private notebook.
         # GAP-A26: always include sources key so consumers have a stable schema.
+        # #973: store ALL urls (no [:3] cap) — full pages captured to Relata below.
         payload["sources"] = []
+        _all_urls: list[str] = []
         if tool_calls:
             with contextlib.suppress(Exception):
                 from hypermind.orgs.evidence import extract_citations
 
-                _urls = [
+                _all_urls = [
                     c.url
                     for c in extract_citations(
                         [{"by": m.label, "tool_calls": [dict(tc) for tc in tool_calls]}]
                     )
                     if c.url
-                ][:3]
-                if _urls:
-                    payload["sources"] = _urls
+                ]
+                if _all_urls:
+                    payload["sources"] = _all_urls
         kid = await m.agent.publish_claim(payload, uncertainty=post, topic=m.unit)
         _log.info(
             "publish: agent=%r topic=%r round=%d conf=%.3f evidenced=%s kid=%s",
@@ -5029,6 +5031,31 @@ class OrgSim:
                         )
                     elif hasattr(_mem_store, "associate"):
                         _mem_store.associate(kid.hex(), _ev_mem_id, "evidence_for")
+        # #973: fetch all search result URLs, convert to markdown, store in Relata memory.
+        # Fire-and-forget — never blocks publish on fetch failure.
+        if _all_urls and self.relata_url and kid is not None:
+            with contextlib.suppress(Exception):
+                from hypermind.orgs.web_capture import capture_and_store_urls
+
+                _agent_id = getattr(m.agent, "kid_hex", None) or m.label
+                _page_pairs = await capture_and_store_urls(
+                    _all_urls,
+                    agent_id=_agent_id,
+                    topic=m.unit,
+                    relata_url=self.relata_url,
+                    relata_token=self.relata_token,
+                    namespace=getattr(self, "_ns", ""),
+                )
+                # link each stored page back to the claim
+                if _page_pairs and _ev_mem_id is not None:
+                    _mem_store = getattr(m.agent, "_memory", None)
+                    if _mem_store is not None and hasattr(_mem_store, "associate_batch"):
+                        _mem_store.associate_batch(
+                            [
+                                {"from_id": kid.hex(), "to_id": mem_id, "relation": "source_for"}
+                                for _, mem_id in _page_pairs
+                            ]
+                        )
         # RELATA-001: also persist evidence KV to Relata so cross-run _retrieve()
         # can surface citations without re-running the LLM.
         if self._backend is not None and kid is not None:

hypermind-2.1.2/src/hypermind/orgs/web_capture.py

@@ -0,0 +1,207 @@
+"""#973: Fetch web search result URLs and store as markdown in Relata memory.
+
+Pipeline per URL:
+  fetch HTML (urllib) → trafilatura → clean markdown → /memory/remember
+  {content, metadata:{url, agent_id, path, title, topic}, tags:[...]}
+
+trafilatura is optional ([ingest] extra). Falls back to bs4 text strip,
+then to snippet-only if neither is installed.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import urllib.request
+
+_log = logging.getLogger(__name__)
+
+_FETCH_TIMEOUT = 10.0
+_MAX_CONTENT_CHARS = 80_000  # ~20k tokens — cap before storing
+
+
+def _html_to_markdown(html: str, url: str) -> tuple[str, str]:
+    """Return (title, markdown). Tries trafilatura first, then bs4, then raw."""
+    # trafilatura: best quality — strips nav/ads/footers, outputs clean markdown
+    try:
+        import trafilatura
+
+        result = trafilatura.extract(
+            html,
+            output_format="markdown",
+            include_links=True,
+            include_tables=True,
+            no_fallback=False,
+        )
+        meta = trafilatura.extract_metadata(html)
+        title = (meta.title or "") if meta else ""
+        if result:
+            return title, result
+    except ImportError:
+        pass
+    except Exception as exc:
+        _log.debug("trafilatura failed for %s: %s", url, exc)
+
+    # bs4 fallback: already optional dep under [ingest]
+    try:
+        from bs4 import BeautifulSoup
+
+        soup = BeautifulSoup(html, "html.parser")
+        title = soup.title.string.strip() if soup.title and soup.title.string else ""
+        for tag in soup(["script", "style", "nav", "footer", "header"]):
+            tag.decompose()
+        text = soup.get_text(separator="\n\n")
+        # Convert to minimal markdown — paragraphs separated by blank lines
+        lines = [ln.strip() for ln in text.splitlines()]
+        md = "\n\n".join(p for p in "\n".join(lines).split("\n\n") if p.strip())
+        return title, md
+    except ImportError:
+        pass
+    except Exception as exc:
+        _log.debug("bs4 fallback failed for %s: %s", url, exc)
+
+    return "", ""
+
+
+def fetch_and_convert(url: str) -> dict[str, str] | None:
+    """Fetch *url* and convert to markdown. Returns None on any failure.
+
+    Returns dict with keys: url, title, markdown.
+    Never raises — callers must not let a failed fetch block publish.
+    """
+    try:
+        req = urllib.request.Request(
+            url,
+            headers={
+                "User-Agent": "HyperMind/2.1 (research agent; +https://hypermind.dev)",
+                "Accept": "text/html,application/xhtml+xml",
+            },
+        )
+        with urllib.request.urlopen(req, timeout=_FETCH_TIMEOUT) as resp:
+            content_type = resp.headers.get("Content-Type", "")
+            if "text/html" not in content_type and "text/plain" not in content_type:
+                return None  # skip binary/PDF/etc
+            html = resp.read().decode("utf-8", errors="replace")
+    except Exception as exc:
+        _log.debug("web_capture: fetch failed url=%s err=%s", url, exc)
+        return None
+
+    title, markdown = _html_to_markdown(html, url)
+    if not markdown.strip():
+        return None
+
+    return {
+        "url": url,
+        "title": title,
+        "markdown": markdown[:_MAX_CONTENT_CHARS],
+    }
+
+
+def url_path(agent_id: str, url: str) -> str:
+    """Stable storage path: `{agent_id}/web-search/{url_sha256[:16]}`."""
+    url_hash = hashlib.sha256(url.encode()).hexdigest()[:16]
+    return f"{agent_id}/web-search/{url_hash}"
+
+
+def store_page(
+    page: dict[str, str],
+    *,
+    agent_id: str,
+    topic: str,
+    relata_url: str,
+    relata_token: str | None,
+    namespace: str = "",
+) -> str | None:
+    """POST page markdown to Relata /memory/remember with full metadata.
+
+    Returns the memory_id on success, None on failure.
+    """
+    path = url_path(agent_id, page["url"])
+    body = {
+        "content": f"# {page['title']}\n\n{page['markdown']}"
+        if page["title"]
+        else page["markdown"],
+        "metadata": {
+            "url": page["url"],
+            "title": page["title"],
+            "agent_id": agent_id,
+            "path": path,
+            "source": "web-search",
+            "topic": topic,
+        },
+        "tags": ["web-search", topic, agent_id],
+        "confidence": 1.0,
+    }
+    try:
+        data = json.dumps(body).encode()
+        headers: dict[str, str] = {"Content-Type": "application/json"}
+        if relata_token:
+            headers["Authorization"] = f"Bearer {relata_token}"
+        if namespace:
+            headers["X-Organization-Id"] = namespace
+        req = urllib.request.Request(
+            f"{relata_url.rstrip('/')}/memory/remember",
+            data=data,
+            headers=headers,
+            method="POST",
+        )
+        with urllib.request.urlopen(req, timeout=10) as resp:
+            result = json.loads(resp.read())
+            return result.get("id")
+    except Exception as exc:
+        _log.debug("web_capture: store failed url=%s err=%s", page["url"], exc)
+        return None
+
+
+async def capture_and_store_urls(
+    urls: list[str],
+    *,
+    agent_id: str,
+    topic: str,
+    relata_url: str,
+    relata_token: str | None,
+    namespace: str = "",
+) -> list[tuple[str, str]]:
+    """Fetch, convert, and store all *urls*. Returns [(url, memory_id)] for successful ones.
+
+    Runs fetches in a thread pool to avoid blocking the event loop.
+    Each URL is independent — failures are silently skipped.
+    """
+    import asyncio
+    import concurrent.futures
+
+    results: list[tuple[str, str]] = []
+    if not urls or not relata_url:
+        return results
+
+    loop = asyncio.get_event_loop()
+
+    def _process(url: str) -> tuple[str, str] | None:
+        page = fetch_and_convert(url)
+        if page is None:
+            return None
+        mem_id = store_page(
+            page,
+            agent_id=agent_id,
+            topic=topic,
+            relata_url=relata_url,
+            relata_token=relata_token,
+            namespace=namespace,
+        )
+        if mem_id:
+            _log.debug("web_capture: stored url=%s mem_id=%s", url, mem_id)
+            return (url, mem_id)
+        return None
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
+        futures = [loop.run_in_executor(pool, _process, url) for url in urls]
+        for fut in asyncio.as_completed(futures):
+            try:
+                r = await fut
+                if r:
+                    results.append(r)
+            except Exception:
+                pass
+
+    return results