clio-kit 2.6.2__tar.gz → 2.6.3__tar.gz

{clio_kit-2.6.2 → clio_kit-2.6.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: clio-kit
-Version: 2.6.2
+Version: 2.6.3
 Summary: CLIO Kit - MCP Servers, Clients, and Tools for AI Agents
 Project-URL: Homepage, https://github.com/iowarp/clio-kit
 Project-URL: Repository, https://github.com/iowarp/clio-kit
@@ -49,6 +49,7 @@ Description-Content-Type: text/markdown
 <!-- mcp-name: io.github.iowarp/slurm-mcp -->
 <!-- mcp-name: io.github.iowarp/spack-mcp -->
 <!-- mcp-name: io.github.iowarp/terrain-mcp -->
+<!-- mcp-name: io.github.iowarp/web-mcp -->
 
 [![License: BSD-3-Clause](https://img.shields.io/badge/License-BSD--3--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause)
 [![PyPI version](https://img.shields.io/pypi/v/clio-kit.svg)](https://pypi.org/project/clio-kit/)
@@ -290,6 +291,7 @@ distinct `not_installed` semantic, while real Spack failures remain errors.
 | **`slurm`** | 3.0.0 | HPC | Job submission and management | `clio-kit mcp-server slurm` |
 | **`spack`** | 2.1.0 | Package Management | Structured package discovery, installation, and location | `clio-kit mcp-server spack` |
 | **`terrain`** | 2.2.3 | Geospatial | Analyze DEMs and terrain point clouds | `clio-kit mcp-server terrain` |
+| **`web`** | 1.0.0 | Web | Fetch a URL to Markdown and search the web | `clio-kit mcp-server web` |
 
 </div>
 

{clio_kit-2.6.2 → clio_kit-2.6.3}/README.md

@@ -23,6 +23,7 @@
 <!-- mcp-name: io.github.iowarp/slurm-mcp -->
 <!-- mcp-name: io.github.iowarp/spack-mcp -->
 <!-- mcp-name: io.github.iowarp/terrain-mcp -->
+<!-- mcp-name: io.github.iowarp/web-mcp -->
 
 [![License: BSD-3-Clause](https://img.shields.io/badge/License-BSD--3--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause)
 [![PyPI version](https://img.shields.io/pypi/v/clio-kit.svg)](https://pypi.org/project/clio-kit/)
@@ -264,6 +265,7 @@ distinct `not_installed` semantic, while real Spack failures remain errors.
 | **`slurm`** | 3.0.0 | HPC | Job submission and management | `clio-kit mcp-server slurm` |
 | **`spack`** | 2.1.0 | Package Management | Structured package discovery, installation, and location | `clio-kit mcp-server spack` |
 | **`terrain`** | 2.2.3 | Geospatial | Analyze DEMs and terrain point clouds | `clio-kit mcp-server terrain` |
+| **`web`** | 1.0.0 | Web | Fetch a URL to Markdown and search the web | `clio-kit mcp-server web` |
 
 </div>
 

{clio_kit-2.6.2 → clio_kit-2.6.3}/clio-kit-mcp-servers/adios/server.json

@@ -12,7 +12,7 @@
     {
       "registryType": "pypi",
       "identifier": "clio-kit",
-      "version": "2.6.2",
+      "version": "2.6.3",
       "transport": {
         "type": "stdio"
       },

{clio_kit-2.6.2 → clio_kit-2.6.3}/clio-kit-mcp-servers/arxiv/server.json

@@ -12,7 +12,7 @@
     {
       "registryType": "pypi",
       "identifier": "clio-kit",
-      "version": "2.6.2",
+      "version": "2.6.3",
       "transport": {
         "type": "stdio"
       },

{clio_kit-2.6.2 → clio_kit-2.6.3}/clio-kit-mcp-servers/chronolog/server.json

@@ -12,7 +12,7 @@
     {
       "registryType": "pypi",
       "identifier": "clio-kit",
-      "version": "2.6.2",
+      "version": "2.6.3",
       "transport": {
         "type": "stdio"
       },

{clio_kit-2.6.2 → clio_kit-2.6.3}/clio-kit-mcp-servers/compression/server.json

@@ -12,7 +12,7 @@
     {
       "registryType": "pypi",
       "identifier": "clio-kit",
-      "version": "2.6.2",
+      "version": "2.6.3",
       "transport": {
         "type": "stdio"
       },

{clio_kit-2.6.2 → clio_kit-2.6.3}/clio-kit-mcp-servers/darshan/server.json

@@ -12,7 +12,7 @@
     {
       "registryType": "pypi",
       "identifier": "clio-kit",
-      "version": "2.6.2",
+      "version": "2.6.3",
       "transport": {
         "type": "stdio"
       },

{clio_kit-2.6.2 → clio_kit-2.6.3}/clio-kit-mcp-servers/geo/server.json

@@ -12,7 +12,7 @@
     {
       "registryType": "pypi",
       "identifier": "clio-kit",
-      "version": "2.6.2",
+      "version": "2.6.3",
       "transport": {
         "type": "stdio"
       },

{clio_kit-2.6.2 → clio_kit-2.6.3}/clio-kit-mcp-servers/geojson/server.json

@@ -12,7 +12,7 @@
     {
       "registryType": "pypi",
       "identifier": "clio-kit",
-      "version": "2.6.2",
+      "version": "2.6.3",
       "transport": {
         "type": "stdio"
       },

{clio_kit-2.6.2 → clio_kit-2.6.3}/clio-kit-mcp-servers/hdf5/server.json

@@ -12,7 +12,7 @@
     {
       "registryType": "pypi",
       "identifier": "clio-kit",
-      "version": "2.6.2",
+      "version": "2.6.3",
       "transport": {
         "type": "stdio"
       },

{clio_kit-2.6.2 → clio_kit-2.6.3}/clio-kit-mcp-servers/jarvis/server.json

@@ -12,7 +12,7 @@
     {
       "registryType": "pypi",
       "identifier": "clio-kit",
-      "version": "2.6.2",
+      "version": "2.6.3",
       "transport": {
         "type": "stdio"
       },

{clio_kit-2.6.2 → clio_kit-2.6.3}/clio-kit-mcp-servers/lmod/server.json

@@ -12,7 +12,7 @@
     {
       "registryType": "pypi",
       "identifier": "clio-kit",
-      "version": "2.6.2",
+      "version": "2.6.3",
       "transport": {
         "type": "stdio"
       },

{clio_kit-2.6.2 → clio_kit-2.6.3}/clio-kit-mcp-servers/ndp/server.json

@@ -12,7 +12,7 @@
     {
       "registryType": "pypi",
       "identifier": "clio-kit",
-      "version": "2.6.2",
+      "version": "2.6.3",
       "transport": {
         "type": "stdio"
       },

{clio_kit-2.6.2 → clio_kit-2.6.3}/clio-kit-mcp-servers/node-hardware/server.json

@@ -12,7 +12,7 @@
     {
       "registryType": "pypi",
       "identifier": "clio-kit",
-      "version": "2.6.2",
+      "version": "2.6.3",
       "transport": {
         "type": "stdio"
       },

{clio_kit-2.6.2 → clio_kit-2.6.3}/clio-kit-mcp-servers/pandas/server.json

@@ -12,7 +12,7 @@
     {
       "registryType": "pypi",
       "identifier": "clio-kit",
-      "version": "2.6.2",
+      "version": "2.6.3",
       "transport": {
         "type": "stdio"
       },

{clio_kit-2.6.2 → clio_kit-2.6.3}/clio-kit-mcp-servers/parallel-sort/server.json

@@ -12,7 +12,7 @@
     {
       "registryType": "pypi",
       "identifier": "clio-kit",
-      "version": "2.6.2",
+      "version": "2.6.3",
       "transport": {
         "type": "stdio"
       },

{clio_kit-2.6.2 → clio_kit-2.6.3}/clio-kit-mcp-servers/paraview/server.json

@@ -12,7 +12,7 @@
     {
       "registryType": "pypi",
       "identifier": "clio-kit",
-      "version": "2.6.2",
+      "version": "2.6.3",
       "transport": {
         "type": "stdio"
       },

{clio_kit-2.6.2 → clio_kit-2.6.3}/clio-kit-mcp-servers/parquet/server.json

@@ -12,7 +12,7 @@
     {
       "registryType": "pypi",
       "identifier": "clio-kit",
-      "version": "2.6.2",
+      "version": "2.6.3",
       "transport": {
         "type": "stdio"
       },

{clio_kit-2.6.2 → clio_kit-2.6.3}/clio-kit-mcp-servers/plot/server.json

@@ -12,7 +12,7 @@
     {
       "registryType": "pypi",
       "identifier": "clio-kit",
-      "version": "2.6.2",
+      "version": "2.6.3",
       "transport": {
         "type": "stdio"
       },

{clio_kit-2.6.2 → clio_kit-2.6.3}/clio-kit-mcp-servers/sac/server.json

@@ -12,7 +12,7 @@
     {
       "registryType": "pypi",
       "identifier": "clio-kit",
-      "version": "2.6.2",
+      "version": "2.6.3",
       "transport": {
         "type": "stdio"
       },

{clio_kit-2.6.2 → clio_kit-2.6.3}/clio-kit-mcp-servers/scientific-catalog/server.json

@@ -12,7 +12,7 @@
     {
       "registryType": "pypi",
       "identifier": "clio-kit",
-      "version": "2.6.2",
+      "version": "2.6.3",
       "transport": {
         "type": "stdio"
       },

{clio_kit-2.6.2 → clio_kit-2.6.3}/clio-kit-mcp-servers/seismic/server.json

@@ -12,7 +12,7 @@
     {
       "registryType": "pypi",
       "identifier": "clio-kit",
-      "version": "2.6.2",
+      "version": "2.6.3",
       "transport": {
         "type": "stdio"
       },

{clio_kit-2.6.2 → clio_kit-2.6.3}/clio-kit-mcp-servers/slurm/server.json

@@ -12,7 +12,7 @@
     {
       "registryType": "pypi",
       "identifier": "clio-kit",
-      "version": "2.6.2",
+      "version": "2.6.3",
       "transport": {
         "type": "stdio"
       },

{clio_kit-2.6.2 → clio_kit-2.6.3}/clio-kit-mcp-servers/spack/server.json

@@ -12,7 +12,7 @@
     {
       "registryType": "pypi",
       "identifier": "clio-kit",
-      "version": "2.6.2",
+      "version": "2.6.3",
       "transport": {
         "type": "stdio"
       },

{clio_kit-2.6.2 → clio_kit-2.6.3}/clio-kit-mcp-servers/terrain/server.json

@@ -12,7 +12,7 @@
     {
       "registryType": "pypi",
       "identifier": "clio-kit",
-      "version": "2.6.2",
+      "version": "2.6.3",
       "transport": {
         "type": "stdio"
       },

clio_kit-2.6.3/clio-kit-mcp-servers/web/.claude-plugin/plugin.json

@@ -0,0 +1,5 @@
+{
+  "name": "clio-web",
+  "description": "Web MCP server providing curated fetch + search tools for agentic web access",
+  "version": "1.0.0"
+}

clio_kit-2.6.3/clio-kit-mcp-servers/web/.mcp.json

@@ -0,0 +1,9 @@
+{
+  "clio-web": {
+    "command": "clio-kit",
+    "args": [
+      "mcp-server",
+      "web"
+    ]
+  }
+}

clio_kit-2.6.3/clio-kit-mcp-servers/web/README.md

@@ -0,0 +1,87 @@
+# Web MCP Server
+
+[![Python](https://img.shields.io/badge/Python-3.10%2B-blue)](https://www.python.org/)
+
+**Part of [CLIO Kit](https://docs.iowarp.ai/) - Gnosis Research Center**
+
+The Web MCP server provides two curated, high-level tools for agentic web
+access. It is the seed of CLIO's web tooling and the test instrument for the
+sandbox campaign's egress recording.
+
+## Tools
+
+### `fetch`
+
+Retrieve an HTTP(S) URL, convert HTML to Markdown, and return it inline or save
+it to a local file.
+
+- **Agent story:** "I found a promising URL and need its actual content." The
+  agent calls `fetch` to pull the page, get clean Markdown (links preserved),
+  and read the title. For large or binary resources it passes `to_file=True` so
+  the raw page is written to disk and only a `local_path` comes back — raw pages
+  never bloat the context window. The agent can then pipe that file through its
+  own model for `(url, prompt)`-style extraction.
+- **Behavior:** streamed download with a hard size cap (default 5 MiB, enforced
+  on `Content-Length` and mid-stream) and configurable timeouts;
+  `follow_redirects=True`; HTML → Markdown via trafilatura, falling back to
+  readability-lxml then a plain-text strip; non-HTML text returned as-is; binary
+  content only materialized when `to_file=True`.
+- **Returns:** `{ok, url, content | local_path, size_bytes, content_type,
+  status, title, method: "http"}`. The `url` key always carries the fetched
+  source URL verbatim — CLIO's provenance layer keys the web source off it.
+
+### `search`
+
+Query a configurable web-search provider and return ranked results.
+
+- **Agent story:** "I need candidate URLs for a question." The agent calls
+  `search` and gets back a small list of `{title, url, snippet}` rows to triage
+  and then `fetch`.
+- **Providers:** keyless **DuckDuckGo** (`ddg`, default, via the `ddgs`
+  package); optional BYO-key **Brave** and **Tavily**. Selecting a keyed
+  provider without its key raises a typed error naming the missing config — it
+  never silently falls back to `ddg`.
+- **Returns:** `{ok, provider, query, results: [{title, url, snippet}], count}`.
+
+## Configuration
+
+Configuration is a single `pydantic-settings` `Settings` model. All fields are
+overridable via `WEB_`-prefixed environment variables or a `.env` file, but the
+recommended path is a single source of config with clear defaults.
+
+| Field | Env var | Default | Meaning |
+| --- | --- | --- | --- |
+| `search_provider` | `WEB_SEARCH_PROVIDER` | `"ddg"` | Active search provider (`ddg` / `brave` / `tavily`). |
+| `brave_api_key` | `WEB_BRAVE_API_KEY` | `None` | Brave Search API key (required for `brave`). |
+| `tavily_api_key` | `WEB_TAVILY_API_KEY` | `None` | Tavily API key (required for `tavily`). |
+| `max_bytes` | `WEB_MAX_BYTES` | `5242880` | Fetch size cap in bytes (5 MiB). |
+| `connect_timeout_s` | `WEB_CONNECT_TIMEOUT_S` | `5.0` | HTTP connect timeout. |
+| `read_timeout_s` | `WEB_READ_TIMEOUT_S` | `30.0` | HTTP read timeout. |
+| `artifacts_root` | `WEB_ARTIFACTS_ROOT` | `None` (→ CWD) | Default directory for `to_file` output. |
+
+## Documented extension points (not yet available in v1)
+
+These are honest, typed gaps — the follow-on campaign's remainder — not silent
+failures:
+
+- **Headless-browser escalation (Playwright)** for JS-rendered / Anubis-walled
+  pages. When HTML extraction yields nothing, `fetch` returns a typed note
+  `reason="js_render_required_browser_unavailable"` instead of returning junk.
+- **`(url, prompt)` small-model extraction.** An MCP server has no model access,
+  so page-specific extraction stays the agent's job: `fetch` with
+  `to_file=True`, then let the agent pipe the saved file through its own model.
+
+## Development
+
+```bash
+uv sync
+uv run pytest -q            # network-mocked suite (default)
+uv run ruff check src/ tests/
+uv run mypy src/
+```
+
+Real-network checks are opt-in and skipped by default:
+
+```bash
+WEB_MCP_LIVE=1 uv run pytest -m integration
+```

clio_kit-2.6.3/clio-kit-mcp-servers/web/docs/web_tools.md

@@ -0,0 +1,34 @@
+# CLIO Web MCP server — tool reference
+
+Two curated tools for agentic web access. Runs as a confined fleet child under CLIO's sandbox,
+so all egress is recorded through the network chokepoint.
+
+## `fetch(url, *, to_file=False, output_dir=None, max_bytes=None, timeout=None)`
+
+Retrieve an HTTP(S) URL. HTML is converted to Markdown (trafilatura → readability → plain-text
+strip; the `extractor` field names which ran, so a quality downgrade is never silent). Returns
+inline `content`, or — with `to_file=True` — writes it under the artifacts root and returns
+`local_path` so raw pages never bloat context.
+
+- **Size cap** (default 5 MiB, `WEB_MAX_BYTES`): enforced on `Content-Length` *and* mid-stream.
+- **SSRF guard** (default on; `WEB_ALLOW_PRIVATE_HOSTS=true` to disable): refuses loopback /
+  private / link-local literal hosts (incl. the cloud-metadata address `169.254.169.254`) and
+  `localhost` names, on the initial URL **and every redirect hop** (redirects are followed
+  manually so each hop is checked before connecting). `final_url` reports the resolved URL.
+- **Typed gaps (honest, never junk):** an empty extraction returns
+  `reason="js_render_required_browser_unavailable"` (headless-browser escalation is the
+  follow-on); binary without `to_file` returns `reason="binary_content_not_inlined"`.
+- **`(url, prompt)` extraction** is the agent's job (an MCP server has no model access): use
+  `to_file=True` and pipe the saved file through the agent's own model.
+
+## `search(query, *, provider=None, count=5)`
+
+Search via a configurable provider. Keyless **DuckDuckGo** (`ddg`) by default; BYO-key **Brave**
+(`WEB_BRAVE_API_KEY`) / **Tavily** (`WEB_TAVILY_API_KEY`). Selecting a keyed provider without its
+key is a typed error — never a silent fallback to `ddg`. `count` is capped at 25. Returns
+`{title, url, snippet}` results.
+
+## Configuration (`WEB_`-prefixed env or a `.env`)
+
+`WEB_SEARCH_PROVIDER`, `WEB_BRAVE_API_KEY`, `WEB_TAVILY_API_KEY`, `WEB_MAX_BYTES`,
+`WEB_CONNECT_TIMEOUT_S`, `WEB_READ_TIMEOUT_S`, `WEB_ARTIFACTS_ROOT`, `WEB_ALLOW_PRIVATE_HOSTS`.

clio_kit-2.6.3/clio-kit-mcp-servers/web/pyproject.toml

@@ -0,0 +1,77 @@
+[project]
+name = "web-mcp"
+version = "1.0.0"
+description = "Web MCP server providing curated fetch + search tools for agentic web access"
+readme = "README.md"
+requires-python = ">=3.10"
+license = "BSD-3-Clause"
+authors = [
+    { name = "IoWarp Team - Gnosis Research Center", email = "grc@illinoistech.edu" }
+]
+
+keywords = ["web", "fetch", "search", "mcp", "llm-integration", "agentic-web"]
+
+dependencies = [
+    "fastmcp>=3.2.0",
+    "httpx>=0.27.0",
+    "trafilatura>=1.12",
+    "readability-lxml>=0.8",
+    "ddgs>=6.0",
+    "pydantic>=2.5",
+    "pydantic-settings>=2.6",
+    "python-dotenv>=1.0",
+]
+
+[project.urls]
+Homepage = "https://docs.iowarp.ai/"
+Repository = "https://github.com/iowarp/clio-kit"
+
+[project.scripts]
+web-mcp = "web_mcp.server:main"
+
+[dependency-groups]
+dev = [
+    "pytest>=8.0.0",
+    "pytest-asyncio>=1.0.0",
+    "pytest-cov>=4.0.0",
+    "pytest-httpx>=0.30.0",
+    "ruff>=0.12.5",
+    "mypy>=1.7.0",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/web_mcp"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+
+[tool.ruff.lint]
+select = ["E", "F", "W", "I", "N", "UP", "S", "B", "A", "C4", "ICN", "PIE", "T20", "Q"]
+ignore = ["S101", "S104", "S603", "S607"]
+
+[tool.mypy]
+python_version = "3.10"
+warn_return_any = true
+warn_unused_configs = true
+disallow_untyped_defs = true
+disallow_incomplete_defs = true
+check_untyped_defs = true
+strict_optional = true
+
+[[tool.mypy.overrides]]
+module = ["trafilatura", "readability", "ddgs"]
+ignore_missing_imports = true
+
+[tool.pytest.ini_options]
+pythonpath = ["src"]
+testpaths = ["tests"]
+asyncio_mode = "auto"
+addopts = "--tb=short"
+markers = [
+    "integration: marks tests that hit the real network (deselect with '-m \"not integration\"')",
+]

clio_kit-2.6.3/clio-kit-mcp-servers/web/server.json

@@ -0,0 +1,55 @@
+{
+  "$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json",
+  "name": "io.github.iowarp/web-mcp",
+  "title": "CLIO Web",
+  "description": "Web MCP server providing curated fetch + search tools for agentic web access",
+  "version": "1.0.0",
+  "repository": {
+    "url": "https://github.com/iowarp/clio-kit",
+    "source": "github"
+  },
+  "packages": [
+    {
+      "registryType": "pypi",
+      "identifier": "clio-kit",
+      "version": "2.6.3",
+      "transport": {
+        "type": "stdio"
+      },
+      "packageArguments": [
+        {
+          "type": "positional",
+          "value": "mcp-server"
+        },
+        {
+          "type": "positional",
+          "value": "web"
+        }
+      ]
+    }
+  ],
+  "tools": [
+    {
+      "name": "fetch",
+      "description": "Fetch an HTTP(S) URL with a streamed size cap and timeout, convert HTML to Markdown, and return the content inline or (to_file=True) write it to a local file and return its path."
+    },
+    {
+      "name": "search",
+      "description": "Search the web via a configurable provider (keyless DuckDuckGo by default; optional BYO-key Brave or Tavily) and return ranked results."
+    }
+  ],
+  "resources": [
+    {
+      "uri": "web://providers",
+      "name": "search_providers",
+      "description": "The active + available web-search providers and current fetch limits."
+    }
+  ],
+  "prompts": [
+    {
+      "name": "research_web",
+      "description": "Guided search then fetch workflow for researching a topic on the web."
+    }
+  ],
+  "tags": []
+}

clio_kit-2.6.3/clio-kit-mcp-servers/web/src/web_mcp/__init__.py

@@ -0,0 +1,12 @@
+"""
+Web MCP Server
+
+This package provides an MCP server for agentic web access with two curated
+tools: ``fetch`` (retrieve a URL and convert HTML to Markdown, optionally to a
+local file) and ``search`` (query a configurable web-search provider).
+
+It is the seed of CLIO's web tooling and the test instrument for the sandbox
+campaign's egress recording.
+"""
+
+__version__ = "1.0.0"