agentcrawl-ai 0.1.0__tar.gz

agentcrawl_ai-0.1.0/.dockerignore

@@ -0,0 +1,15 @@
+.venv
+__pycache__
+*.pyc
+.pytest_cache
+.ruff_cache
+agentcrawl.db
+.git
+.env
+tests
+examples
+.github
+PROJECT_STATE.md
+PROGRESS_LOG.md
+VPS_DEPLOYMENT.md
+README_AGENT_CONNECTION.md

agentcrawl_ai-0.1.0/.env.example

@@ -0,0 +1,28 @@
+# AgentCrawl server
+AGENTCRAWL_AUTH_ENABLED=true
+AGENTCRAWL_ALLOW_LOCAL_FILES=false
+AGENTCRAWL_ALLOW_PRIVATE_NETWORK=false
+AGENTCRAWL_API_KEYS=change-me-dev-key
+AGENTCRAWL_OWNER_API_KEYS=change-me-dev-key
+AGENTCRAWL_DB=/data/agentcrawl.db
+AGENTCRAWL_FETCHER=http
+AGENTCRAWL_HEADLESS=true
+AGENTCRAWL_TIMEOUT_MS=30000
+AGENTCRAWL_HTTP_RETRIES=2
+AGENTCRAWL_HTTP_RETRY_DELAY=1.0
+AGENTCRAWL_BROWSER_FALLBACK=false
+AGENTCRAWL_DOMAIN_MIN_DELAY=0.35
+AGENTCRAWL_DOMAIN_MAX_CONCURRENCY=2
+AGENTCRAWL_CACHE_ENABLED=true
+AGENTCRAWL_CACHE_TTL_SECONDS=86400
+AGENTCRAWL_CRAWL_DEPTH=1
+AGENTCRAWL_CRAWL_MAX_PAGES=25
+AGENTCRAWL_RESPECT_ROBOTS_TXT=true
+AGENTCRAWL_WORKERS=4
+AGENTCRAWL_CRAWL_JOB_PAGE_QUANTUM=5
+AGENTCRAWL_RATE_LIMIT_PER_MINUTE=60
+AGENTCRAWL_USER_AGENT=Mozilla/5.0 (compatible; AgentCrawl/0.1; +https://agentcrawl.local)
+
+# Remote client / MCP server
+AGENTCRAWL_BASE_URL=http://127.0.0.1:8000
+AGENTCRAWL_API_KEY=change-me-dev-key

agentcrawl_ai-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,42 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.12"]
+
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+      - name: Install
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e ".[server,mcp,dev]"
+      - name: Ruff
+        run: python -m ruff check agentcrawl tests examples benchmarks
+      - name: Format
+        run: python -m ruff format --check agentcrawl tests examples benchmarks
+      - name: Tests
+        run: python -m pytest -q
+      - name: Compile
+        run: python -m compileall -q agentcrawl benchmarks
+
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - run: docker build -t agentcrawl-community:ci .

agentcrawl_ai-0.1.0/.github/workflows/docker.yml

@@ -0,0 +1,68 @@
+name: Build and Push Docker Image
+
+on:
+  push:
+    branches: [main]
+    tags: ['v*']
+  pull_request:
+    branches: [main]
+  workflow_dispatch:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to Container Registry
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=sha,prefix=,suffix=,format=short
+            type=raw,value=latest,enable={{is_default_branch}}
+
+      - name: Build local image for smoke test
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          load: true
+          tags: agentcrawl:smoke
+
+      - name: Smoke test Docker image
+        run: |
+          docker run --rm agentcrawl:smoke agentcrawl --version
+          docker run --rm agentcrawl:smoke agentcrawl doctor
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

agentcrawl_ai-0.1.0/.gitignore

@@ -0,0 +1,13 @@
+.venv/
+__pycache__/
+*.py[cod]
+.pytest_cache/
+.ruff_cache/
+*.db
+.env
+.DS_Store
+
+# Build artifacts
+build/
+dist/
+*.egg-info/

agentcrawl_ai-0.1.0/.hermes/plans/2026-06-11_000405-agentcrawl-docs-cleanup.md

@@ -0,0 +1,239 @@
+# Plan — limpieza y refinado de documentación AgentCrawl Community
+
+Fecha: 2026-06-11
+Repo público: `/home/orgej/Proyectos/agentcrawl`
+Docs privados: `/home/orgej/Proyectos/agentcrawl-private-docs`
+Estado base verificado:
+
+```text
+Branch: main
+Estado: limpio y sincronizado con origin/main
+HEAD: ea897d3 Improve README conversion quickstart
+CI: success
+Docker/GHCR workflow: success
+GHCR latest/main: ea897d3
+VPS: no tocar; despliegue sigue aplazado
+```
+
+## Objetivo
+
+Antes del Bloque 6 de ejemplos/cookbook, limpiar y alinear todos los Markdown para que el repo público parezca un producto Community serio y no un acumulado de fases internas.
+
+Resultado buscado:
+
+- Public docs coherentes entre sí.
+- Nada privado/comercial sensible dentro del repo público.
+- README fuerte pero no demasiado largo ni contradictorio.
+- Release/roadmap/examples actualizados al estado real: PyPI preparado, GHCR ya publicado, Docker ligero, Phase 2 quality verde.
+- Private handoff actualizado con el estado real posterior a los commits `9060705` y `ea897d3`.
+
+## Hallazgos iniciales
+
+### 1. Repo/GHCR están actualizados
+
+- `origin/main` está en `ea897d3`.
+- GitHub CI y Docker workflow verdes.
+- GHCR contiene tags `latest`, `main`, `ea897d3`.
+
+### 2. Drift privado importante
+
+Los docs privados siguen diciendo que el estado canónico era:
+
+```text
+1b0d460 Add browser rendered SPA quality coverage
+```
+
+Pero el estado real ahora es:
+
+```text
+ea897d3 Improve README conversion quickstart
+9060705 Prepare lightweight GHCR Docker image
+```
+
+Archivos privados afectados:
+
+- `/home/orgej/Proyectos/agentcrawl-private-docs/INDEX.md`
+- `/home/orgej/Proyectos/agentcrawl-private-docs/HANDOFF.md`
+- `/home/orgej/Proyectos/agentcrawl-private-docs/SESSION_CONTINUITY_2026-06-10.md`
+
+### 3. Public docs tienen wording viejo o inconsistente
+
+Puntos detectados:
+
+- `ROADMAP.md` aún lista “public GHCR image” como prioridad pendiente, aunque ya está publicado.
+- `CHANGELOG.md` no menciona claramente Docker ligero, GHCR publicado, README conversion quickstart ni fixture count actual.
+- `docs/RELEASE.md` describe GHCR como “expected public image path after publication”; ahora ya está publicado.
+- `docs/EXAMPLES.md` mezcla ejemplos actuales con una sección “Additional Examples On The Roadmap” que justo coincide con Bloque 6; hay que convertir eso en archivos reales o lista limpia de pendientes.
+- `docs/OPERATIONS.md` usa `docker compose up --build -d`; para usuarios públicos debería preferir la imagen GHCR y dejar build local como opción de desarrollo.
+- `docs/EXAMPLES.md` tiene placeholders tipo `$AGENT...KEY` / `Bearer` que conviene normalizar para evitar filtrados raros y confusión.
+- `COMMUNITY_VS_ENHANCED.md` en el repo público aparece vacío. Hay que decidir: eliminarlo si no está trackeado/relevante, o llenarlo con una versión pública mínima sin estrategia privada.
+
+### 4. README ya mejoró, pero necesita un pase de consistencia
+
+El README tiene buena conversión inicial, pero antes de seguir hay que revisar:
+
+- longitud y orden;
+- placeholders de API key;
+- que Docker use GHCR como camino principal;
+- que no prometa browser fallback dentro de la imagen default;
+- que “Community includes” no regale Enhanced ni prometa cloud.
+
+## Plan por bloques
+
+### Bloque A — Normalización pública mínima
+
+Archivos:
+
+- `README.md`
+- `ROADMAP.md`
+- `CHANGELOG.md`
+- `docs/RELEASE.md`
+- `docs/OPERATIONS.md`
+- `docs/QUALITY_BENCHMARKS.md`
+- `docs/COMPARISON.md`
+
+Acciones:
+
+1. Actualizar ROADMAP:
+   - Mover GHCR publicado a Completed.
+   - Dejar pendiente solo PyPI publication, examples/cookbook, live smoke targets y docs finales.
+   - No hablar de “Phase 2” como si fuera interno; usar “Extraction quality hardening”.
+
+2. Actualizar CHANGELOG:
+   - Añadir GHCR lightweight Docker image.
+   - Añadir Docker workflow smoke tests.
+   - Añadir README conversion quickstart.
+   - Añadir quality report 14 fixtures / threshold 85 / avg green sin claim competitivo.
+
+3. Actualizar RELEASE:
+   - Cambiar “expected image path” por “published image path”.
+   - Incluir verificación GHCR:
+     ```bash
+     docker pull ghcr.io/jorg18/agentcrawl:latest
+     docker run --rm ghcr.io/jorg18/agentcrawl:latest agentcrawl --version
+     docker run --rm ghcr.io/jorg18/agentcrawl:latest agentcrawl doctor
+     ```
+   - Mantener PyPI como pendiente si no hay token/publicación final.
+
+4. Actualizar OPERATIONS:
+   - Primero GHCR image.
+   - Luego Compose local/dev.
+   - Dejar `docker compose up --build` como desarrollo, no como instalación principal.
+
+5. Revisar COMPARISON:
+   - Asegurar que no parece que ya tenemos todo lo de competidores.
+   - Mantener tono: “fits when…” sin claims no probados.
+
+6. Revisar QUALITY_BENCHMARKS:
+   - Confirmar que refleja 14 fixtures.
+   - Explicar que benchmarks competitivos quedan diferidos.
+
+### Bloque B — Examples/cookbook preparación antes de crear ejemplos reales
+
+Archivos actuales/potenciales:
+
+- `docs/EXAMPLES.md`
+- `examples/basic.py`
+- `examples/graph_extraction.py`
+- nuevos propuestos:
+  - `examples/cli_scrape.md`
+  - `examples/python_scrape.py`
+  - `examples/api_scrape.md`
+  - `examples/docker.md`
+  - `examples/mcp.md`
+  - `examples/browser_rendered.md`
+
+Acciones:
+
+1. Convertir `docs/EXAMPLES.md` en índice/cookbook principal.
+2. Mover los ejemplos largos a archivos concretos.
+3. Mantener 5-6 ejemplos buenos, no 20.
+4. Cada ejemplo debe ser:
+   - copy-pasteable;
+   - probado si es ejecutable;
+   - claramente documental si requiere API/MCP externa.
+5. Evitar `python -m pip install -e` en docs para usuarios finales salvo sección “from repo checkout”.
+
+Validación:
+
+- Ejecutar `python examples/python_scrape.py` si usa una fixture/local file o `https://example.com`.
+- Smoke CLI contra `https://example.com`.
+- Si API example requiere server, arrancar servidor temporal o documentarlo claramente.
+
+### Bloque C — Private handoff synchronization
+
+Archivos privados:
+
+- `/home/orgej/Proyectos/agentcrawl-private-docs/INDEX.md`
+- `/home/orgej/Proyectos/agentcrawl-private-docs/HANDOFF.md`
+- `/home/orgej/Proyectos/agentcrawl-private-docs/SESSION_CONTINUITY_2026-06-10.md`
+
+Acciones:
+
+1. Actualizar estado canónico:
+   - HEAD `ea897d3`.
+   - GHCR publicado.
+   - README conversion quickstart hecho.
+   - Docker local ya no es requisito; GitHub Actions construye y publica.
+2. Mantener política VPS:
+   - No tocar VPS aún.
+   - Despliegue dedicado con backup/smoke test más adelante.
+3. Actualizar “next recommended work”:
+   - limpieza docs/refino;
+   - Bloque 6 examples/cookbook;
+   - docs finales;
+   - release tag/PyPI si hay token.
+
+### Bloque D — Verificación final antes de commit
+
+Ejecutar:
+
+```bash
+.venv/bin/python -m compileall -q agentcrawl benchmarks
+.venv/bin/python -m pytest -q
+.venv/bin/ruff check agentcrawl tests benchmarks examples
+.venv/bin/ruff format --check agentcrawl tests benchmarks examples
+.venv/bin/python benchmarks/quality_report.py
+```
+
+Además:
+
+```bash
+git diff --stat
+git diff --check
+gh run list --repo JorG18/agentcrawl --limit 5
+```
+
+Si se toca Docker/GHCR docs solamente, no hace falta reconstruir imagen salvo que cambie Dockerfile/workflow.
+
+### Bloque E — Commit/push limpio
+
+Commit sugerido tras Bloque A/C:
+
+```bash
+git add README.md ROADMAP.md CHANGELOG.md docs/*.md /home/orgej/Proyectos/agentcrawl-private-docs/*.md
+git commit -m "Refine AgentCrawl Community release docs"
+git push origin main
+```
+
+Nota: los docs privados no están en el repo público, así que probablemente se actualizan sin commit público. Reportar claramente:
+
+- “Repo público commit X”.
+- “Docs privados actualizados localmente”.
+
+## Riesgos / cuidado
+
+- No copiar estrategia privada de monetización al repo público.
+- No prometer hosted/Enhanced como ya disponible.
+- No publicar benchmarks competitivos aún.
+- No tocar VPS.
+- No reinstalar herramientas ni modificar Hermes.
+- No sobreescribir MDs enteros sin re-leerlos completos, salvo README/plan donde se controle el diff.
+
+## Orden recomendado de ejecución
+
+1. Bloque A: limpiar docs públicos de estado/distribución.
+2. Bloque C: sincronizar docs privados con estado real.
+3. Validación rápida.
+4. Commit/push si todo está limpio.
+5. Después sí: Bloque 6 examples/cookbook.

agentcrawl_ai-0.1.0/AGENTS.md

@@ -0,0 +1,3 @@
+# Agent Instructions
+
+When asked to install, configure, connect, test, or use AgentCrawl, read and execute [INSTALL_FOR_AGENTS.md](INSTALL_FOR_AGENTS.md). That file is the canonical procedure. Do not merely describe the commands when the user requested installation.

agentcrawl_ai-0.1.0/CHANGELOG.md

@@ -0,0 +1,19 @@
+# Changelog
+
+## 0.1.0 - Unreleased
+
+- Initial AgentCrawl Community release candidate.
+- CLI, Python library, HTTP API, Docker/GHCR image, and MCP integration.
+- Local and HTTP scraping with optional browser/Camofox fallback.
+- Main-content Markdown extraction with semantic container selection, text-rich fallback blocks, and boilerplate reduction.
+- Markdown table preservation and fenced code blocks with language tags from common HTML classes.
+- Local document ingestion for Markdown, text, JSON, XML/RSS/Atom, and PDF-to-Markdown through the optional `docs` extra.
+- Mapping, crawling, persistent jobs, progress, cancellation, event history, crawl failures, and selective failure retries.
+- Cache controls, usage reporting, operational stats, backup, and restore.
+- Authentication, SSRF protections, unsafe redirect blocking, private-network controls, and safe server defaults.
+- Safety baseline fixes for text normalization, sitemap discovery, PDF limits, scrape error behavior, URL validation, and crawl failure filtering.
+- Package version export, wheel/sdist build verification, `twine check`, and clean install smoke tests for base/server/MCP/docs extras.
+- Lightweight default Docker image based on `python:3.12-slim`, published through GHCR as `ghcr.io/jorg18/agentcrawl:latest`.
+- GitHub Actions Docker workflow builds, smoke-tests, and publishes GHCR images for `main`, tags, and commit SHAs.
+- README quickstart refreshed around CLI, Python, MCP, Docker/API, and Community scope.
+- Quality report baseline: 14 checked-in fixtures, minimum score threshold 85, current local average 100.0, richer provenance metadata, JSON-LD/Product schema extraction, Markdown structure metrics, and noisy-layout coverage.

agentcrawl_ai-0.1.0/CLAUDE.md

@@ -0,0 +1,3 @@
+# Claude Code Instructions
+
+For AgentCrawl installation, MCP registration, verification, and normal tool-selection rules, read and execute [INSTALL_FOR_AGENTS.md](INSTALL_FOR_AGENTS.md). Preserve unrelated client configuration and never expose credentials.

agentcrawl_ai-0.1.0/CONTRIBUTING.md

@@ -0,0 +1,10 @@
+# Contributing
+
+1. Create a focused branch.
+2. Keep changes scoped and avoid unrelated refactors.
+3. Add tests for behavior changes.
+4. Run `pytest -q` and `ruff check agentcrawl tests examples`.
+5. Do not commit credentials, deployment addresses, databases, scraped content, or private commercial modules.
+6. Explain behavior, security impact, and verification in the pull request.
+
+By contributing, you agree that your contribution is licensed under Apache License 2.0.

agentcrawl_ai-0.1.0/Dockerfile

@@ -0,0 +1,35 @@
+FROM python:3.12-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    AGENTCRAWL_DB=/data/agentcrawl.db \
+    AGENTCRAWL_FETCHER=http \
+    AGENTCRAWL_AUTH_ENABLED=true \
+    AGENTCRAWL_ALLOW_LOCAL_FILES=false \
+    AGENTCRAWL_ALLOW_PRIVATE_NETWORK=false
+
+WORKDIR /app
+
+LABEL org.opencontainers.image.title="AgentCrawl" \
+      org.opencontainers.image.description="Self-hosted web extraction and Markdown crawling for AI agents." \
+      org.opencontainers.image.source="https://github.com/JorG18/agentcrawl" \
+      org.opencontainers.image.licenses="Apache-2.0"
+
+COPY pyproject.toml README.md LICENSE ./
+COPY agentcrawl ./agentcrawl
+
+RUN python -m pip install --no-cache-dir --upgrade pip \
+    && python -m pip install --no-cache-dir -e ".[server,mcp]" \
+    && useradd --create-home --uid 10001 agentcrawl \
+    && mkdir -p /data \
+    && chown -R agentcrawl:agentcrawl /app /data
+
+USER agentcrawl
+
+VOLUME ["/data"]
+EXPOSE 8000
+
+HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
+    CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=3).read()"
+
+CMD ["uvicorn", "agentcrawl.server:app", "--host", "0.0.0.0", "--port", "8000"]

agentcrawl_ai-0.1.0/INSTALL_FOR_AGENTS.md

@@ -0,0 +1,170 @@
+# Install AgentCrawl For An Agent Client
+
+This file is the canonical installation procedure for coding agents. When a user asks you to inspect this repository, install AgentCrawl, connect it to your tools, or make it the default scraper, execute the procedure. Do not only explain commands.
+
+## Goal
+
+Install AgentCrawl, verify direct scraping, register its standards-based stdio MCP server in your own client configuration, reload when required, and verify `scrape_url`.
+
+## Safety Rules
+
+- Never print API keys or copy secrets into chat.
+- Do not overwrite unrelated MCP servers or client settings.
+- Back up a configuration file before modifying it.
+- Prefer the client own supported MCP command or structured configuration format.
+- Do not install Playwright browsers unless the user requests browser fallback or the environment needs it. HTTP extraction works without a browser runtime.
+
+## 1. Install
+
+For a published release:
+
+```bash
+python -m pip install "agentcrawl-ai[browser]"
+```
+
+For a repository checkout before a release is published:
+
+```bash
+python -m pip install "agentcrawl-ai[browser]"
+```
+
+The base package uses HTTP and does not install a browser. Install other capabilities only when needed:
+
+```bash
+python -m pip install "agentcrawl-ai[browser]"
+python -m pip install "agentcrawl-ai[docs]"     # local PDF ingestion
+python -m pip install "agentcrawl-ai[browser]"
+playwright install chromium
+```
+
+Inspect the installation:
+
+```bash
+agentcrawl doctor
+```
+
+`doctor` reports installed extras, Python/command discovery, local scrape health,
+and optional remote API health when `AGENTCRAWL_BASE_URL` is set. It only reports
+whether an API key is configured and never prints secret values.
+
+## 2. Verify Direct Scraping
+
+```bash
+agentcrawl scrape https://example.com
+```
+
+Success requires non-empty content containing `Example Domain`. Fix installation or network errors before configuring MCP.
+
+## 3. Register The MCP Server
+
+Current stdio launcher:
+
+```text
+command: agentcrawl
+args: ["mcp"]
+```
+
+Equivalent generic MCP configuration:
+
+```json
+{
+  "mcpServers": {
+    "agentcrawl": {
+      "command": "agentcrawl",
+      "args": ["mcp"]
+    }
+  }
+}
+```
+
+Use your own client supported registration mechanism. Inspect its existing configuration or CLI help instead of guessing a path. Preserve all unrelated settings.
+
+Without `AGENTCRAWL_BASE_URL`, the MCP server runs the local HTTP scraper directly and needs no separate API process. If AgentCrawl is a remote HTTP service, set environment variables on the MCP server process:
+
+```json
+{
+  "mcpServers": {
+    "agentcrawl": {
+      "command": "agentcrawl",
+      "args": ["mcp"],
+      "env": {
+        "AGENTCRAWL_BASE_URL": "https://agentcrawl.example.com",
+        "AGENTCRAWL_API_KEY": "<secret>"
+      }
+    }
+  }
+}
+```
+
+Store real secrets using the client credential mechanism or protected environment files. Never commit them.
+
+## 4. Reload And Verify
+
+Reload or restart your client if it does not hot-reload MCP configuration. Verify that the `agentcrawl` server exposes at least:
+
+```text
+scrape_url
+map_site
+crawl_site
+get_job
+cancel_job
+job_events
+inspect_failures
+retry_failures
+usage
+cache_stats
+clear_cache
+```
+
+Call `scrape_url` with:
+
+```json
+{
+  "url": "https://example.com",
+  "formats": ["markdown", "metadata"]
+}
+```
+
+Success requires clean Markdown containing `Example Domain`.
+
+## 5. Normal Tool Selection
+
+After registration:
+
+- Use `scrape_url` for one known URL.
+- Use `map_site` to discover site URLs without scraping all pages.
+- Use `crawl_site` for bounded multi-page extraction.
+- For asynchronous crawl jobs, provide a stable idempotency key, keep the returned `job_id`, and poll `get_job`; do not start duplicates.
+- A queued job with a future `available_at` is waiting for persisted backoff, not stuck.
+- Read large completed crawls page by page with `offset` and `limit` until `has_more` is false.
+- Use browser automation only for interactive actions, not ordinary page reading.
+- Use another extractor only after AgentCrawl returns a definitive error.
+
+## Operator Backup
+
+Before deploying over an existing server database, run:
+
+```bash
+agentcrawl backup --db /path/to/agentcrawl.db --output-dir /path/to/backups
+```
+
+Use `--env-file /path/to/agentcrawl.env` only when an environment file should be
+copied into the backup directory. Do not print or paste its contents.
+
+Restore only with the service stopped and only from a verified backup:
+
+```bash
+agentcrawl restore --backup-db /path/to/backup.db --db /path/to/agentcrawl.db --force
+```
+
+## Completion Report
+
+Report only:
+
+- installation method and version;
+- whether direct scraping succeeded;
+- whether MCP registration succeeded;
+- number of AgentCrawl tools discovered;
+- whether the functional `scrape_url` test succeeded.
+
+Do not include credentials.