pro-fetch-mcp 0.0.0-stage → 0.3.0

package/.env.example

@@ -0,0 +1,119 @@
+# pro-fetch-mcp — environment reference
+#
+# Copy to `.env` or (recommended on Windows) set these as USER environment variables.
+# NEVER commit real secrets. Every secret below may alternatively be supplied as
+# <NAME>_FILE=<path to a local file containing only the value> (max 16 KiB).
+# Empty values mean "not configured".
+
+# ── Profile / tool visibility ────────────────────────────────────────────────
+# smart (5 meta-tools, default in mcp.json) | minimal | standard (default) | full
+PRO_FETCH_PROFILE=smart
+
+# ── Safety policy ────────────────────────────────────────────────────────────
+# true => block every mutating tool and every non-GET/HEAD/OPTIONS HTTP method
+PRO_FETCH_READ_ONLY=
+# Comma-separated tool names allowed even in read-only mode
+PRO_FETCH_MUTATION_ALLOWLIST=
+# Comma-separated policy lists (tool names / category names)
+PRO_FETCH_ALLOWED_TOOLS=
+PRO_FETCH_DENIED_TOOLS=
+PRO_FETCH_DENIED_CATEGORIES=
+# Files written via output_path / storage_state_path must live under this directory
+PRO_FETCH_OUTPUT_ROOT=
+# Required to run `npm test` / `npm build` tools (still needs confirm=true per call)
+PRO_FETCH_ALLOW_COMMANDS=
+# Required for browser evaluate/execute/script tools (run arbitrary page JavaScript)
+PRO_FETCH_ALLOW_JS=
+# Relax network hardening (leave unset unless you know you need them)
+PRO_FETCH_ALLOW_HTTPS_DOWNGRADE=
+PRO_FETCH_ALLOW_UNPINNED_PROXY_DNS=
+
+# ── Limits ───────────────────────────────────────────────────────────────────
+PRO_FETCH_TOOL_TIMEOUT_MS=180000
+# After a tool timeout, how long its concurrency slots stay reserved while the
+# handler is still running before they are force-released (default 30000)
+PRO_FETCH_ABANDON_GRACE_MS=30000
+PRO_FETCH_MAX_CONCURRENCY=32
+PRO_FETCH_MAX_CATEGORY_CONCURRENCY=8
+PRO_FETCH_MAX_NETWORK_CONCURRENCY=32
+PRO_FETCH_MAX_PER_HOST_CONCURRENCY=4
+PRO_FETCH_MAX_INPUT_BYTES=2097152
+PRO_FETCH_MAX_BYTES=
+PRO_FETCH_TIMEOUT_MS=
+PRO_FETCH_COMMAND_TIMEOUT_MS=30000
+PRO_FETCH_BROWSER_MAX_SESSIONS=
+PRO_FETCH_GITHUB_MAX_PAGES=5
+
+# ── Paths ────────────────────────────────────────────────────────────────────
+PRO_FETCH_CACHE_DIR=.pro-fetch-cache
+PRO_FETCH_HTTP_CACHE_TTL_MS=
+# Root for local git/devops/database/storage tools (defaults to the process cwd)
+PRO_FETCH_WORKSPACE=
+PRO_FETCH_SQLITE_PATH=
+PRO_FETCH_KNOWLEDGE_DB=
+# Playwright build to drive: chrome | msedge | chromium | firefox | webkit
+# Empty = Playwright's own bundled build.
+PRO_FETCH_BROWSER_CHANNEL=
+# IANA timezone applied to browser contexts, for example Europe/Tashkent.
+# Empty = the host timezone.
+PRO_FETCH_BROWSER_TZ=
+
+# ── Proxy (standard variables are honoured) ──────────────────────────────────
+HTTP_PROXY=
+HTTPS_PROXY=
+ALL_PROXY=
+NO_PROXY=
+
+# ── Credentials (leave empty in version control) ─────────────────────────────
+# Any credential may instead be supplied as <NAME>_FILE=<path>. A relative path is
+# resolved against the package root, not the process working directory, so it keeps
+# working regardless of where the MCP client spawns the server. `.secrets/` is
+# git-ignored and is the recommended location for local secret files.
+#
+#   JIRA_TOKEN_FILE=.secrets/JIRA_TOKEN.txt
+#
+# GitHub (REST tools, code search, release monitoring)
+GITHUB_TOKEN=
+GITHUB_CLIENT_ID=
+# Web search (Serper/Google yoki Brave API; ikkalasi ham bo'sh bo'lsa DuckDuckGo'ga tushadi)
+SERPER_API_KEY=
+SEARCH_API_KEY=
+SERPAPI_API_KEY=
+YOUTUBE_API_KEY=
+
+# OpenAI-compatible endpoint for embeddings / media tools
+OPENAI_API_KEY=
+OPENAI_BASE_URL=https://api.openai.com/v1
+OPENAI_EMBEDDINGS_BASE_URL=
+OPENAI_EMBEDDING_MODEL=
+
+# Code hosts (MAX codehost pack)
+GITLAB_TOKEN=
+GITEA_TOKEN=
+FORGEJO_TOKEN=
+BITBUCKET_TOKEN=
+
+# Collaboration
+SLACK_BOT_TOKEN=
+LINEAR_API_KEY=
+JIRA_BASE_URL=
+JIRA_EMAIL=
+JIRA_TOKEN=
+NOTION_TOKEN=
+
+# Storage / productivity
+DROPBOX_TOKEN=
+GOOGLE_DRIVE_TOKEN=
+MICROSOFT_GRAPH_TOKEN=
+GOOGLE_ACCESS_TOKEN=
+GMAIL_TOKEN=
+TODOIST_API_TOKEN=
+
+# Databases (MAX database pack; connection strings contain secrets)
+DATABASE_URL=
+POSTGRES_URL=
+MYSQL_URL=
+REDIS_URL=
+# Optional exact host allowlist for self-hosted GitLab/Gitea/Forgejo/Bitbucket APIs.
+# Comma-separated hostnames only, without scheme or path. Empty means official defaults only.
+PRO_FETCH_ALLOWED_CODEHOST_HOSTS=

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Shuhratbek
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

package/README.md

@@ -1,3 +1,158 @@
-# Temporary Holding Version
+# pro-fetch-mcp
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+Professional fetch, deep-research & GitHub MCP server — **571 MCP tool, 26 categories, one security engine**.
+
+A model gets the tools it actually needs: the default `smart` profile exposes **10 meta-tools** that compose all 571, so any MCP client works without drowning in schemas.
+
+![npm version](https://img.shields.io/npm/v/pro-fetch-mcp?label=npm)
+![node](https://img.shields.io/badge/node-%3E%3D22.13-blue)
+![license](https://img.shields.io/npm/l/pro-fetch-mcp)
+![tests](https://img.shields.io/badge/tests-242%20passing-brightgreen)
+
+## Why
+
+- **571 working tools** — fetch, crawl, extract, convert, OCR, search, research, git, GitHub, GitLab/Gitea/Forgejo/Bitbucket, SQLite, security, Slack/Jira/Linear/Notion, storage, and more. Every one is a real implementation, not a stub.
+- **Smart System** — 10 meta-tools (`pro_capability_call`, `pro_capability_map`, `pro_research_deep`, …) that dispatch to the full catalog in a single round trip. Batch execution, BM25 task-to-tool discovery, and a `detail` mode that cut a 17-step audit from 43 KB to 3.8 KB (**91% token savings**, measured).
+- **Security is the engine, not a warning** — SSRF guard with DNS pinning, workspace sandbox with symlink tracking, per-call permission policy, credential stripping on cross-origin redirects, HTTPS→HTTP downgrade blocking. Every composite call passes through the same `dispatch()` gate.
+- **Degrades, never dies** — native binaries (sharp, canvas, tesseract) are lazy-loaded; missing git/docker/kubectl produce a clear `COMMAND_NOT_FOUND`; the server boots and serves 500+ tools on a host with none of them.
+
+## Quick start
+
+```bash
+npx -y pro-fetch-mcp        # stdio MCP server, zero config
+```
+
+Any MCP client — Claude Desktop, Cursor, Qwen, VS Code, your own harness:
+
+```json
+{
+  "mcpServers": {
+    "pro-fetch": {
+      "command": "npx",
+      "args": ["-y", "pro-fetch-mcp"],
+      "env": { "PRO_FETCH_PROFILE": "smart" }
+    }
+  }
+}
+```
+
+Ready-to-copy config: [`mcp.example.json`](mcp.example.json).
+
+Credentials go in a `.env` next to the server (or `*_FILE` env vars pointing at a file — never the value itself). `.env` is never committed and never ships in the npm package.
+
+| Credential | Unlocks |
+|---|---|
+| `GITHUB_TOKEN` | code search, private repos, higher rate limits |
+| `SERPER_API_KEY` | Google-backed web search |
+| `OPENAI_API_KEY` | audio transcription, embedding hybrid retrieval |
+| `SEARCH_API_KEY` | Brave search (DuckDuckGo fallback otherwise) |
+| `GITLAB_TOKEN`, `SLACK_BOT_TOKEN`, `JIRA_TOKEN`, … | the matching adapter pack — see [`.env.example`](.env.example) |
+
+## The Smart System
+
+Most MCP clients choke on hundreds of tool schemas. `smart` (the default) shows the model **10 tools** that compose everything:
+
+| Tool | What it does |
+|---|---|
+| `pro_capability_call` | **Batch executor** — runs a list of `{tool, arguments}` in one call; each result returns its own `ok`/`error` |
+| `pro_capability_map` | Plain-language task → exact tools with their schemas (BM25 + morphological expansion) |
+| `pro_fetch_tool_execute` | Call any tool by exact name |
+| `pro_research_deep` | Search → read → facts → verify → report |
+| `pro_fetch_ingest` | Any URL → clean structured data (page, PDF, CSV, feed) |
+| `pro_audit_page` | One page → every relevant extraction |
+| `pro_repo_audit` | Repo → full report (metadata, README, tree, issues, PRs, releases) |
+| `pro_pr_review` | PR → full review (diff, checks, reviews, comments) |
+| `pro_site_scan` | Crawl a site and explain it |
+| `pro_data_pipeline` | Load, profile and transform a dataset |
+
+Three round trips become one:
+
+```jsonc
+{ "tool": "pro_capability_call", "arguments": {
+  "calls": [
+    { "tool": "pro_fetch_web_search",      "arguments": { "query": "RAG architecture 2025" }, "label": "search" },
+    { "tool": "pro_fetch_academic_search", "arguments": { "query": "retrieval augmented generation" }, "label": "papers" },
+    { "tool": "pro_fetch_news_search",     "arguments": { "query": "RAG news" }, "label": "news" }
+  ]
+}}
+```
+
+**Security is not weakened by composition.** Every internal call goes through the central `dispatch()`, so `PRO_FETCH_ALLOWED_TOOLS`, `PRO_FETCH_READ_ONLY`, the SSRF check, the output sandbox and timeouts apply unchanged. If `pro_capability_call` itself is not on the allowlist, it does not run.
+
+## Tool catalog
+
+<details>
+<summary><b>All 26 categories — 571 tools</b></summary>
+
+| Category | Tools | Category | Tools |
+|---|---:|---|---:|
+| github | 49 | crawl | 22 |
+| browser (Playwright) | 41 | search | 20 |
+| convert | 30 | http | 19 |
+| collaboration (Slack/Linear/Jira/Notion) | 28 | knowledge (local store) | 18 |
+| research | 26 | productivity (Gmail/Calendar/Todoist) | 18 |
+| data | 26 | storage (S3/Drive/Dropbox/OneDrive) | 16 |
+| codehost (GitLab/Gitea/Forgejo/Bitbucket) | 25 | media (sharp/EXIF/QR/OCR/PDF) | 15 |
+| extract (CSS/XPath/JSONPath/OG) | 24 | archive | 15 |
+| devops (docker/kubectl/terraform/npm) | 24 | docs (arXiv/npm/PyPI/RFC/PubMed) | 15 |
+| database (SQLite/Postgres/MySQL/Redis) | 24 | auth (sessions & keys, in-memory only) | 12 |
+| security (OSV/secret scan/SBOM) | 24 | apitest (OpenAPI/GraphQL/WSDL) | 12 |
+| admin (discovery, metrics, policy) | 23 | i18n | 12 |
+| git | 23 | monitoring | 10 |
+
+</details>
+
+**Profiles** control how many schemas the model sees — not the security boundary:
+
+| Profile | Tools | Contents |
+|---|---:|---|
+| `minimal` | 72 | http, convert, admin |
+| `standard` (default) | 177 | + github, search, extract, auth |
+| `smart` | 10 | the Smart System above |
+| `full` | 571 | everything |
+
+Real access control lives in the environment: `PRO_FETCH_ALLOWED_TOOLS`, `PRO_FETCH_DENIED_TOOLS`, `PRO_FETCH_DENIED_CATEGORIES`, `PRO_FETCH_READ_ONLY`, `PRO_FETCH_MUTATION_ALLOWLIST`, `PRO_FETCH_OUTPUT_ROOT` — enforced on every call, including calls made through the meta-executor.
+
+## Security model
+
+- **SSRF guard** — outbound HTTP(S) resolves and verifies DNS, pins the connection; `HTTP_PROXY`/`HTTPS_PROXY` require an explicit opt-in because a remote proxy resolves DNS itself.
+- **Workspace sandbox** — with `PRO_FETCH_OUTPUT_ROOT`, every `output_path` and browser `storage_state_path` write is contained (symlinks and junctions are followed and rejected on escape).
+- **GitHub input guard** — `owner`/`repo`/`path`/`gist_id`/SHAs are validated centrally; traversal, deep paths and API-relevant characters are rejected.
+- **Redirect safety** — credentials are stripped on cross-origin hops; HTTPS→HTTP downgrades are blocked; redirect *loops* are reported as bot protection, not as a hop count.
+- **Process hardening** — media bytes are identified by signature *before* any decoder sees them (a hostile file cannot crash a worker thread), every `import().then()` chain carries a `.catch()`, and `uncaughtException`/`unhandledRejection` handlers log to stderr instead of killing the server.
+- **Secrets** — diagnostics report only whether a credential is present and its source, never its value.
+
+Full boundary list: [SECURITY.md](SECURITY.md).
+
+## Requirements & graceful degradation
+
+**Node.js ≥ 22.13** (required by `pdfjs-dist@6`). Everything else is optional:
+
+| Optional | Effect when absent |
+|---|---|
+| `sharp`, `@napi-rs/canvas`, `tesseract.js` binaries | media tools fail individually; server boots and serves the rest |
+| `git`, `docker`, `kubectl`, `terraform`, `psql`, `redis-cli` | those tools answer `COMMAND_NOT_FOUND` with a precise message |
+| Playwright browsers | browser pack reports it needs `npx playwright install --with-deps chromium` |
+| npm build scripts (`--ignore-scripts`) | SQLite tools report `SQLITE_UNAVAILABLE` with the fix (`npm rebuild better-sqlite3`) |
+
+SQLite needs **no system binary** — it uses the bundled `better-sqlite3` driver (the `sqlite3` CLI is only a fallback). On Windows, npm tools invoke `npm-cli.js` through `process.execPath`, so `npm audit`, `npm outdated` and license reports work there too.
+
+## Development
+
+```bash
+npm ci
+npm run check   # verify:static → verify:registry → build → test → verify:package → format:check
+```
+
+- **242 tests** (Vitest): SSRF, redirects, permission policy, output sandbox, timeouts, every tool's input contract, and live provider-failure behaviour.
+- `verify:registry` boots the real registry and asserts **571 tools / 26 categories / all four profile counts**.
+- `verify:package` proves the published tarball has the shebang, the `files` allowlist, and **zero credentials or source files**.
+- The MCP protocol itself was exercised live: `initialize`, `tools/list` (571/571 unique, valid schemas), `tools/call` for every smart tool, hidden tools refused, clean JSON-RPC on stdout.
+
+## License
+
+[MIT](LICENSE)
+
+---
+
+[O'zbekcha versiya](README.uz.md) · [Changelog](CHANGELOG.md) · [Contributing](CONTRIBUTING.md)

package/README.uz.md

@@ -0,0 +1,158 @@
+# pro-fetch-mcp
+
+Professional fetch, deep-research va GitHub MCP serveri — **571 tool, 26 kategoriya, bitta xavfsizlik enginei**.
+
+Modelga kerak bo'lgan tool beriladi: standart `smart` profil **10 ta meta-tool** orqali barcha 571 ni tarkibiy tarzda ochadi — MCP mijoz schema'lar botinishiga yo'l qo'ymaydi.
+
+![npm version](https://img.shields.io/npm/v/pro-fetch-mcp?label=npm)
+![node](https://img.shields.io/badge/node-%3E%3D22.13-blue)
+![license](https://img.shields.io/npm/l/pro-fetch-mcp)
+![tests](https://img.shields.io/badge/tests-242%20passing-brightgreen)
+
+## Nima uchun
+
+- **571 ishlaydigan tool** — fetch, crawl, ekstraksiya, konvertatsiya, OCR, qidiruv, tadqiqot, git, GitHub, GitLab/Gitea/Forgejo/Bitbucket, SQLite, xavfsizlik, Slack/Jira/Linear/Notion, storage va yana. Har biri haqiqiy amalga oshirilgan, stub emas.
+- **Smart System** — 10 ta meta-tool (`pro_capability_call`, `pro_capability_map`, `pro_research_deep`, …) barcha katalogni bitta round trip'da ishga tushiradi. Batch ijro, BM25 vazifa→tool discovery va `detail` rejimi 17-qadamli auditni 43 KB dan 3.8 KB ga qisqartirdi (**o'lchangan: 91% token tejash**).
+- **Xavfsizlik — ogohlantirish emas, engine** — DNS pinning'li SSRF himoyasi, symlink kuzatiladigan workspace sandboxi, har bir chaqiruvda permission siyosati, cross-origin redirect'larda credential stripping, HTTPS→HTTP downgrade blokirovkasi. Har bir kompozit chaqiruv bir xil `dispatch()` darvozasidan o'tadi.
+- **Nobud bo'lmaydi** — native binary'lar (sharp, canvas, tesseract) lazy yuklanadi; git/docker/kubectl yo'qligida aniq `COMMAND_NOT_FOUND`; hech biri o'rnatilmagan hostda ham server ishga tushib 500+ tool xizmat qiladi.
+
+## Tez boshlash
+
+```bash
+npx -y pro-fetch-mcp        # stdio MCP server, nol konfiguratsiya
+```
+
+Istalgan MCP mijoz — Claude Desktop, Cursor, Qwen, VS Code, sizning harness'ingiz:
+
+```json
+{
+  "mcpServers": {
+    "pro-fetch": {
+      "command": "npx",
+      "args": ["-y", "pro-fetch-mcp"],
+      "env": { "PRO_FETCH_PROFILE": "smart" }
+    }
+  }
+}
+```
+
+Tayyor nusxa: [`mcp.example.json`](mcp.example.json).
+
+Kalitlar server yonidagi `.env` da (yoki qiymat o'rnini ko'rsatuvchi `*_FILE` o'zgaruvchilarda) saqlanadi. `.env` hech qachon commit qilinmaydi va npm paketiga kirmaydi.
+
+| Credential | Nima ochadi |
+|---|---|
+| `GITHUB_TOKEN` | code search, private repo, yuqori rate limit |
+| `SERPER_API_KEY` | Google asosli web search |
+| `OPENAI_API_KEY` | audio transkripsiya, embedding hybrid retrieval |
+| `SEARCH_API_KEY` | Brave search (yo'q bo'lsa DuckDuckGo) |
+| `GITLAB_TOKEN`, `SLACK_BOT_TOKEN`, `JIRA_TOKEN`, … | mos adapter pack — [`.env.example`](.env.example) |
+
+## Smart System
+
+Ko'p MCP mijoz yuzlab tool schema'iga chidamaydi. `smart` (standart) modelga **10 ta tool** ko'rsatadi, ular hammasini tarkib qiladi:
+
+| Tool | Nima qiladi |
+|---|---|
+| `pro_capability_call` | **Batch executor** — `{tool, arguments}` ro'yxatini bitta chaqiruvda bajaradi; natijada har biri alohida `ok`/`error` |
+| `pro_capability_map` | Oddiy tildagi vazifa → aniq tool va uning schema'si (BM25 + morfologik kengaytirish) |
+| `pro_fetch_tool_execute` | Aniq nom bilan bitta tool chaqirish |
+| `pro_research_deep` | Qidiruv → o'qish → faktlar → tekshiruv → hisobot |
+| `pro_fetch_ingest` | Istalgan URL → toza tuzilgan ma'lumot (sahifa, PDF, CSV, feed) |
+| `pro_audit_page` | Bitta sahifa → barcha tegishli ekstraksiyalar |
+| `pro_repo_audit` | Repo → to'liq hisobot (metadata, README, tree, issues, PR, release) |
+| `pro_pr_review` | PR → to'liq review (diff, checks, reviews, izohlar) |
+| `pro_site_scan` | Saytni crawl qilib tushuntirish |
+| `pro_data_pipeline` | Datasetni yuklash, profillash va qayta ishlash |
+
+3 ta round trip 1 taga aylanadi:
+
+```jsonc
+{ "tool": "pro_capability_call", "arguments": {
+  "calls": [
+    { "tool": "pro_fetch_web_search",      "arguments": { "query": "RAG arxitekturasi 2025" }, "label": "search" },
+    { "tool": "pro_fetch_academic_search", "arguments": { "query": "retrieval augmented generation" }, "label": "papers" },
+    { "tool": "pro_fetch_news_search",     "arguments": { "query": "RAG news" }, "label": "news" }
+  ]
+}}
+```
+
+**Tarkibiyat xavfsizlikni zaiflashtirmaydi.** Har bir ichki chaqiruv markaziy `dispatch()` orqali o'tadi, shuning uchun `PRO_FETCH_ALLOWED_TOOLS`, `PRO_FETCH_READ_ONLY`, SSRF tekshiruvi, output sandboxi va timeout o'zgarmagan holda qo'llaniladi. `pro_capability_call` ozi allowlistda bo'lmasa, u umuman ishga tushmaydi.
+
+## Tool katalogi
+
+<details>
+<summary><b>Barcha 26 kategoriya — 571 tool</b></summary>
+
+| Kategoriya | Tool | Kategoriya | Tool |
+|---|---:|---|---:|
+| github | 49 | crawl | 22 |
+| browser (Playwright) | 41 | search | 20 |
+| convert | 30 | http | 19 |
+| collaboration (Slack/Linear/Jira/Notion) | 28 | knowledge (local store) | 18 |
+| research | 26 | productivity (Gmail/Calendar/Todoist) | 18 |
+| data | 26 | storage (S3/Drive/Dropbox/OneDrive) | 16 |
+| codehost (GitLab/Gitea/Forgejo/Bitbucket) | 25 | media (sharp/EXIF/QR/OCR/PDF) | 15 |
+| extract (CSS/XPath/JSONPath/OG) | 24 | archive | 15 |
+| devops (docker/kubectl/terraform/npm) | 24 | docs (arXiv/npm/PyPI/RFC/PubMed) | 15 |
+| database (SQLite/Postgres/MySQL/Redis) | 24 | auth (session va kalitlar, faqat xotirada) | 12 |
+| security (OSV/secret scan/SBOM) | 24 | apitest (OpenAPI/GraphQL/WSDL) | 12 |
+| admin (discovery, metrics, policy) | 23 | i18n | 12 |
+| git | 23 | monitoring | 10 |
+
+</details>
+
+**Profillar** modelga qancha schema ko'rinishini boshqaradi — xavfsizlik chegarasi emas:
+
+| Profil | Tool | Tarkibi |
+|---|---:|---|
+| `minimal` | 72 | http, convert, admin |
+| `standard` (default) | 177 | + github, search, extract, auth |
+| `smart` | 10 | yuqoridagi Smart System |
+| `full` | 571 | hammasi |
+
+Haqiqiy kirish nazorati environment'da: `PRO_FETCH_ALLOWED_TOOLS`, `PRO_FETCH_DENIED_TOOLS`, `PRO_FETCH_DENIED_CATEGORIES`, `PRO_FETCH_READ_ONLY`, `PRO_FETCH_MUTATION_ALLOWLIST`, `PRO_FETCH_OUTPUT_ROOT` — har bir chaqiruvda, meta-executor orqali chaqirilganlarda ham, qo'llaniladi.
+
+## Xavfsizlik modeli
+
+- **SSRF himoyasi** — chiquvchi HTTP(S) DNS'ni resolve qilib tekshiradi va connection'ni pin'laydi; `HTTP_PROXY`/`HTTPS_PROXY` faqat ataylab yoqiladi, chunki uzoq proxy target DNS'ini o'zi resolve qiladi.
+- **Workspace sandboxi** — `PRO_FETCH_OUTPUT_ROOT` berilganda har bir `output_path` va browser `storage_state_path` yozuvi shu papka ichida saqlanadi (symlink va junction'lar kuzatilib, qochish rad etiladi).
+- **GitHub kiritma himoyasi** — `owner`/`repo`/`path`/`gist_id`/SHA markaziy ravishda tekshiriladi; traversal, chuqur path va API'ni o'zgartiradigan belgilar rad etiladi.
+- **Redirect xavfsizligi** — cross-origin o'tishlarda credential'lar o'chiriladi; HTTPS→HTTP downgrade bloklanadi; redirect **loop'lari** bot himoyasi sifatida xabarlanadi, hop soni emas.
+- **Jarayon barqarorligi** — media baytlari dekoder ko'rishidan oldin belgi bilan aniqlanadi (dushman fayl worker thread'ni ag'dara olmaydi), har bir `import().then()` zanjirida `.catch()` bor, `uncaughtException`/`unhandledRejection` handler'lari serverni o'ldirish o'rniga stderr'ga yozadi.
+- **Secretlar** — diagnostika faqat credential bor-yo'qligi va manbasini ko'rsatadi, qiymatini yo'q.
+
+To'liq chegara ro'yxati: [SECURITY.md](SECURITY.md).
+
+## Talablar va yumshoq degradatsiya
+
+**Node.js ≥ 22.13** (`pdfjs-dist@6` shuni talab qiladi). Qolgan barchasi ixtiyoriy:
+
+| Ixtiyoriy | Yo'q bo'lgandagi ta'sir |
+|---|---|
+| `sharp`, `@napi-rs/canvas`, `tesseract.js` binary'lari | media tool'lari alohida xato beradi; server ishlayveradi |
+| `git`, `docker`, `kubectl`, `terraform`, `psql`, `redis-cli` | o'sha tool'lar aniq xabar bilan `COMMAND_NOT_FOUND` |
+| Playwright brauzerlari | browser pack `npx playwright install --with-deps chromium` kerakligini aytadi |
+| npm build skriptlari (`--ignore-scripts`) | SQLite tool'lari `SQLITE_UNAVAILABLE` va tuzatish buyrug'ini (`npm rebuild better-sqlite3`) aytadi |
+
+SQLite uchun **sistem binary kerak emas** — paket ichidagi `better-sqlite3` drayver ishlatiladi (`sqlite3` CLI faqat zaxira). Windows'da npm tool'lari `npm-cli.js` ni `process.execPath` orqali chaqiradi, shuning uchun `npm audit`, `npm outdated` va litsensiya hisoboti u yerda ham ishlaydi.
+
+## Dasturlash
+
+```bash
+npm ci
+npm run check   # verify:static → verify:registry → build → test → verify:package → format:check
+```
+
+- **242 ta test** (Vitest): SSRF, redirectlar, permission siyosati, output sandboxi, timeout'lar, har bir tool'ing kiritma shartnomasi va jonli provider-failure xatti-harakati.
+- `verify:registry` haqiqiy registry'ni yuklab **571 tool / 26 kategoriya / barcha 4 profil sonini** tekshiradi.
+- `verify:package` nashr paketida shebang, `files` allowlist va **nol credential/manba fayl** ekanligini isbotlaydi.
+- MCP protokoli o'zi jonli sinovdan o'tkazildi: `initialize`, `tools/list` (571/571 unique, valid schemalar), har bir smart tool uchun `tools/call`, yashirin tool'lar rad etildi, stdout toza JSON-RPC.
+
+## Litsenziya
+
+[MIT](LICENSE)
+
+---
+
+[English version](README.md) · [O'zgarishlar](CHANGELOG.md) · [Hissa qo'shish](CONTRIBUTING.md)

package/dist/core/archive-monitoring-tools.js

@@ -0,0 +1,152 @@
+import { getCredential } from './credentials.js';
+import { createHash } from 'node:crypto';
+import { load } from 'cheerio';
+import { cache } from './cache.js';
+import { fetchUrl } from './fetcher.js';
+import { fetchWithOptions } from './http-client.js';
+import { assertSafeUrl } from './ssrf-guard.js';
+import { result } from './format.js';
+import { parseUpstreamJson } from './json-response.js';
+function str(input, ...keys) {
+    for (const key of keys)
+        if (typeof input[key] === 'string' && String(input[key]).trim())
+            return String(input[key]).trim();
+    return '';
+}
+function list(input, ...keys) {
+    for (const key of keys) {
+        const value = input[key];
+        if (Array.isArray(value))
+            return value;
+        if (typeof value === 'string' && value.trim()) {
+            try {
+                const parsed = JSON.parse(value);
+                if (Array.isArray(parsed))
+                    return parsed;
+            }
+            catch {
+                return value.split(/\r?\n/).filter(Boolean);
+            }
+        }
+    }
+    return [];
+}
+function keyFor(name, id) {
+    return `monitor:${name}:${createHash('sha256').update(id).digest('hex').slice(0, 24)}`;
+}
+async function fetchSnapshot(url) {
+    const safe = await assertSafeUrl(url);
+    const response = await fetchWithOptions(safe.toString());
+    return { status: response.status, text: response.text, finalUrl: safe.toString() };
+}
+export async function runArchiveMonitoringTool(toolName, input) {
+    const name = toolName.replace(/^pro_fetch_/, '');
+    const url = str(input, 'url');
+    if (name === 'archive_org_fetch' || name === 'google_cache_fetch') {
+        if (!url)
+            return result('url parametri kerak.');
+        await assertSafeUrl(url);
+        const archiveUrl = name === 'google_cache_fetch'
+            ? 'https://webcache.googleusercontent.com/search?q=cache:' + encodeURIComponent(url)
+            : 'https://web.archive.org/web/2/' + encodeURIComponent(url);
+        try {
+            const snapshot = await fetchUrl(archiveUrl);
+            if (snapshot.status < 400)
+                return result(`# Archived copy\n\nSource: ${url}\nArchive URL: ${snapshot.url}\n\n${snapshot.markdown}`);
+        }
+        catch {
+            /* try Wayback fallback below */
+        }
+        if (name === 'google_cache_fetch') {
+            const fallback = await fetchUrl('https://web.archive.org/web/2/' + encodeURIComponent(url));
+            return result(`# Wayback fallback (Google cache is unavailable)\n\n${fallback.markdown}`);
+        }
+        throw new Error('Internet Archive’dan snapshot topilmadi.');
+    }
+    if (name === 'archive_org_save') {
+        if (!url)
+            return result('url parametri kerak.');
+        const safe = await assertSafeUrl(url);
+        const response = await fetchWithOptions('https://web.archive.org/save/' + encodeURIComponent(safe.toString()), {
+            method: 'POST',
+            timeoutMs: 60000,
+            retries: 0,
+        });
+        if (response.status >= 400)
+            throw new Error('Wayback save HTTP ' + response.status + ': ' + response.text.slice(0, 500));
+        const archiveUrl = response.headers['content-location'] ??
+            response.headers.location ??
+            'https://web.archive.org/web/' + encodeURIComponent(safe.toString());
+        return result(JSON.stringify({ source: safe.toString(), status: response.status, archiveUrl }, null, 2));
+    }
+    if (name === 'digest_build') {
+        const items = list(input, 'items', 'sources', 'results').map((item) => typeof item === 'string' ? item : JSON.stringify(item));
+        const text = str(input, 'text', 'content');
+        const body = [...items, ...text.split(/\r?\n/)].filter(Boolean);
+        const selected = [...new Set(body)].slice(0, Number(input.limit ?? 30));
+        return result(`# Digest${str(input, 'title') ? ': ' + str(input, 'title') : ''}\n\n${selected.map((item) => '- ' + item.slice(0, 1200)).join('\n') || 'Digest uchun items yoki text bering.'}`);
+    }
+    if (name === 'feed_subscribe' || name === 'jobs_watch' || name === 'keyword_alert' || name === 'price_watch') {
+        if (!url)
+            return result('url parametri kerak.');
+        const fetched = await fetchSnapshot(url);
+        const id = str(input, 'id', 'name', 'selector') || url;
+        const key = keyFor(name, id);
+        let value = fetched.text;
+        if (name === 'feed_subscribe') {
+            const $ = load(fetched.text, { xmlMode: true });
+            value = JSON.stringify($('item,entry')
+                .toArray()
+                .slice(0, 50)
+                .map((item) => ({
+                title: $(item).find('title').first().text().trim(),
+                link: $(item).find('link').first().attr('href') ?? $(item).find('link').first().text().trim(),
+                date: $(item).find('pubDate,published,updated').first().text().trim(),
+            })));
+        }
+        else if (name === 'price_watch') {
+            const $ = load(fetched.text);
+            const selector = str(input, 'selector');
+            const selected = selector ? $(selector).first().text() : fetched.text;
+            const match = selected.replace(/,/g, '').match(/-?\d+(?:\.\d+)?/);
+            value = match?.[0] ?? '';
+            if (!value)
+                return result('Narx raqami topilmadi; selector yoki regex bering.');
+        }
+        else if (name === 'keyword_alert') {
+            const keywords = list(input, 'keywords', 'terms').map(String);
+            const lower = fetched.text.toLowerCase();
+            value = JSON.stringify(keywords.filter((keyword) => lower.includes(keyword.toLowerCase())));
+        }
+        const previous = cache.get(key)?.value;
+        cache.put(key, value, Number(input.ttl_ms ?? 7 * 86_400_000), { url: fetched.finalUrl, status: fetched.status });
+        return result(JSON.stringify({
+            type: name,
+            id,
+            url: fetched.finalUrl,
+            status: fetched.status,
+            changed: previous !== undefined && previous !== value,
+            previous: previous?.slice(0, 2000),
+            current: value.slice(0, 5000),
+            initialized: previous === undefined,
+        }, null, 2));
+    }
+    if (name === 'release_watch') {
+        const repository = str(input, 'repository', 'repo') || (url.match(/github\.com\/([^/]+\/[^/]+)/)?.[1] ?? '');
+        if (!repository || !repository.includes('/'))
+            return result('repository owner/repo yoki GitHub URL kerak.');
+        const [owner, repo] = repository.replace(/\.git$/, '').split('/');
+        const headers = { accept: 'application/vnd.github+json' };
+        if (getCredential('GITHUB_TOKEN'))
+            headers.authorization = 'Bearer ' + getCredential('GITHUB_TOKEN');
+        const response = await fetchWithOptions(`https://api.github.com/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/releases/latest`, { headers });
+        if (response.status >= 400)
+            throw new Error('GitHub release API HTTP ' + response.status);
+        const release = parseUpstreamJson(response, 'GitHub release API');
+        const key = keyFor(name, repository);
+        const previous = cache.get(key)?.value;
+        cache.put(key, String(release.tag_name ?? ''), Number(input.ttl_ms ?? 7 * 86_400_000));
+        return result(JSON.stringify({ repository, ...release, changed: previous !== undefined && previous !== release.tag_name, previous }, null, 2));
+    }
+    return result('Archive/monitoring operation not recognized: ' + toolName);
+}

package/dist/core/binary-source.js

@@ -0,0 +1,23 @@
+import { downloadToBuffer } from './http-client.js';
+import { assertSafeUrl } from './ssrf-guard.js';
+export async function readBinarySource(input, label, defaultMaxBytes = 100 * 1024 * 1024) {
+    const maxBytesValue = Number(input.max_bytes ?? defaultMaxBytes);
+    const maxBytes = Number.isFinite(maxBytesValue)
+        ? Math.max(1, Math.min(defaultMaxBytes, Math.floor(maxBytesValue)))
+        : defaultMaxBytes;
+    const encoded = typeof input.base64 === 'string' ? input.base64 : typeof input.data_base64 === 'string' ? input.data_base64 : '';
+    if (encoded) {
+        const normalized = encoded.replace(/\s+/g, '');
+        if (!/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(normalized))
+            throw new Error(`${label}: base64 qiymati noto‘g‘ri.`);
+        const bytes = Buffer.from(normalized, 'base64');
+        if (bytes.byteLength > maxBytes)
+            throw new Error(`${label}: fayl max_bytes limitdan katta.`);
+        return bytes;
+    }
+    const rawUrl = typeof input.url === 'string' ? input.url.trim() : '';
+    if (!rawUrl)
+        throw new Error(`${label}: url yoki base64 kerak.`);
+    const safeUrl = await assertSafeUrl(rawUrl);
+    return (await downloadToBuffer(safeUrl.toString(), maxBytes, Number(input.timeout_ms ?? 30_000))).bytes;
+}

package/dist/core/bounded-env.js

@@ -0,0 +1,21 @@
+/**
+ * Numeric environment parsing with a bounded range.
+ *
+ * This lives apart from `security-policy.ts` on purpose. `security-policy` needs to read the
+ * active profile from `loader.ts`, and `loader.ts` imports every tool index, which imports
+ * `core/tool.ts`, which imports `execution-engine.ts` — which needs `boundedEnv`. Keeping
+ * `boundedEnv` inside `security-policy` therefore closed an import cycle and made the module
+ * evaluate to `undefined` at load time ("boundedEnv is not a function").
+ *
+ * `boundedEnv` has no dependency on the policy rules, so it belongs in its own leaf module that
+ * both `security-policy` and the concurrency gates can import without creating a cycle.
+ */
+/** Read a positive integer from the environment, clamped to `[min, max]`.
+ *
+ * Returns `fallback` when the variable is unset, non-numeric, zero, or negative — so a typo in
+ * `.env` degrades to a safe default instead of disabling a limit.
+ */
+export function boundedEnv(name, fallback, min, max) {
+    const parsed = Number(process.env[name]);
+    return Number.isFinite(parsed) && parsed > 0 ? Math.max(min, Math.min(Math.floor(parsed), max)) : fallback;
+}