aeo-cli 0.2.1__tar.gz

aeo_cli-0.2.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024-2026 AEO-CLI Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

aeo_cli-0.2.1/PKG-INFO

@@ -0,0 +1,302 @@
+Metadata-Version: 2.4
+Name: aeo-cli
+Version: 0.2.1
+Summary: Agentic Engine Optimization CLI — audit URLs for AI crawler readiness
+Author: Hansel Wahjono
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/hanselhansel/aeo-cli
+Project-URL: Repository, https://github.com/hanselhansel/aeo-cli
+Project-URL: Issues, https://github.com/hanselhansel/aeo-cli/issues
+Keywords: aeo,seo,ai,llm,crawler,audit,robots-txt,schema-org
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: typer>=0.9
+Requires-Dist: rich>=13.0
+Requires-Dist: httpx>=0.27
+Requires-Dist: beautifulsoup4>=4.12
+Requires-Dist: pydantic>=2.0
+Requires-Dist: crawl4ai>=0.4
+Requires-Dist: fastmcp>=2.0
+Provides-Extra: generate
+Requires-Dist: litellm>=1.40; extra == "generate"
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.23; extra == "dev"
+Requires-Dist: pytest-cov>=5.0; extra == "dev"
+Requires-Dist: mypy>=1.10; extra == "dev"
+Requires-Dist: ruff>=0.4; extra == "dev"
+Requires-Dist: litellm>=1.40; extra == "dev"
+Dynamic: license-file
+
+# AEO-CLI
+
+[![Tests](https://github.com/hanselhansel/aeo-cli/actions/workflows/test.yml/badge.svg)](https://github.com/hanselhansel/aeo-cli/actions/workflows/test.yml)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://www.python.org/downloads/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE)
+[![PyPI version](https://img.shields.io/pypi/v/aeo-cli.svg)](https://pypi.org/project/aeo-cli/)
+
+**Audit any URL for AI crawler readiness. Get a 0-100 AEO score.**
+
+## What is AEO?
+
+Agentic Engine Optimization (AEO) is the practice of making your website discoverable and accessible to AI agents and LLM-powered search engines. As AI crawlers like GPTBot, ClaudeBot, and PerplexityBot become major traffic sources, AEO ensures your content is structured, permitted, and optimized for these systems.
+
+AEO-CLI checks how well a URL is prepared for AI consumption and returns a structured score.
+
+## Features
+
+- **Robots.txt AI bot access** — checks whether major AI crawlers are allowed or blocked
+- **llms.txt presence** — detects the emerging standard for LLM-specific site instructions
+- **Schema.org JSON-LD** — extracts and evaluates structured data markup
+- **Content density** — measures useful content vs. boilerplate via markdown conversion
+- **Rich CLI output** — formatted tables and scores via Rich
+- **JSON output** — machine-readable results for pipelines
+- **MCP server** — expose the audit as a tool for AI agents via FastMCP
+- **AEO Compiler** — LLM-powered `llms.txt` and `schema.jsonld` generation
+- **CI/CD integration** — `--fail-under` threshold, `--fail-on-blocked-bots`, GitHub Step Summary
+- **GitHub Action** — composite action for CI pipelines
+
+## Installation
+
+```bash
+pip install aeo-cli
+```
+
+AEO-CLI uses a headless browser for content extraction. After installing, run:
+
+```bash
+crawl4ai-setup
+```
+
+### Development install
+
+```bash
+git clone https://github.com/your-org/aeo-cli.git
+cd aeo-cli
+pip install -e ".[dev]"
+crawl4ai-setup
+```
+
+## Quick Start
+
+```bash
+aeo-cli audit example.com
+```
+
+This runs a full audit and prints a Rich-formatted report with your AEO score.
+
+## CLI Usage
+
+### Single Page Audit
+
+Audit only the specified URL (skip multi-page discovery):
+
+```bash
+aeo-cli audit example.com --single
+```
+
+### Multi-Page Site Audit (default)
+
+Discover pages via sitemap/spider and audit up to 10 pages:
+
+```bash
+aeo-cli audit example.com
+```
+
+### Limit Pages
+
+```bash
+aeo-cli audit example.com --max-pages 5
+```
+
+### JSON Output
+
+Get structured JSON for CI pipelines, dashboards, or scripting:
+
+```bash
+aeo-cli audit example.com --json
+```
+
+### CSV / Markdown Output
+
+```bash
+aeo-cli audit example.com --format csv
+aeo-cli audit example.com --format markdown
+```
+
+### Verbose Mode
+
+Show detailed per-pillar breakdown with scoring explanations:
+
+```bash
+aeo-cli audit example.com --single --verbose
+```
+
+### CI Mode
+
+Fail the build if the AEO score is below a threshold:
+
+```bash
+aeo-cli audit example.com --fail-under 60
+```
+
+Fail if any AI bot is blocked:
+
+```bash
+aeo-cli audit example.com --fail-on-blocked-bots
+```
+
+Exit codes: 0 = pass, 1 = score below threshold, 2 = bots blocked.
+
+When running in GitHub Actions, a markdown summary is automatically written to `$GITHUB_STEP_SUMMARY`.
+
+### Quiet Mode
+
+Suppress output, exit code 0 if score >= 50, 1 otherwise:
+
+```bash
+aeo-cli audit example.com --quiet
+```
+
+Use `--fail-under` with `--quiet` to override the default threshold:
+
+```bash
+aeo-cli audit example.com --quiet --fail-under 70
+```
+
+### Start MCP server
+
+```bash
+aeo-cli mcp
+```
+
+Launches a FastMCP stdio server exposing the audit as a tool for AI agents.
+
+## MCP Integration
+
+To use AEO-CLI as a tool in Claude Desktop, add this to your Claude Desktop config (`claude_desktop_config.json`):
+
+```json
+{
+  "mcpServers": {
+    "aeo-cli": {
+      "command": "aeo-cli",
+      "args": ["mcp"]
+    }
+  }
+}
+```
+
+Once configured, Claude can call the `audit_url` tool directly to check any URL's AEO readiness.
+
+## AEO Compiler (Generate)
+
+Generate `llms.txt` and `schema.jsonld` files from any URL using LLM analysis:
+
+```bash
+pip install aeo-cli[generate]
+aeo-cli generate example.com
+```
+
+This crawls the URL, sends the content to an LLM, and writes optimized files to `./aeo-output/`.
+
+### BYOK (Bring Your Own Key)
+
+The generate command auto-detects your LLM provider from environment variables:
+
+| Priority | Env Variable | Model Used |
+|----------|-------------|------------|
+| 1 | `OPENAI_API_KEY` | gpt-4o-mini |
+| 2 | `ANTHROPIC_API_KEY` | claude-3-haiku-20240307 |
+| 3 | Ollama running locally | ollama/llama3.2 |
+
+Override with `--model`:
+
+```bash
+aeo-cli generate example.com --model gpt-4o
+```
+
+### Industry Profiles
+
+Tailor the output with `--profile`:
+
+```bash
+aeo-cli generate example.com --profile saas
+aeo-cli generate example.com --profile ecommerce
+```
+
+Available: `generic`, `cpg`, `saas`, `ecommerce`, `blog`.
+
+## GitHub Action
+
+Use AEO-CLI in your CI pipeline:
+
+```yaml
+- name: Run AEO Audit
+  uses: hanselhansel/aeo-cli@main
+  with:
+    url: 'https://your-site.com'
+    fail-under: '60'
+```
+
+The action sets up Python, installs aeo-cli, and runs the audit. Outputs `score` and `report-json` for downstream steps. See [docs/ci-integration.md](docs/ci-integration.md) for full documentation.
+
+## Score Breakdown
+
+AEO-CLI returns a score from 0 to 100, composed of four pillars:
+
+| Pillar | Max Points | What it measures |
+|---|---|---|
+| Content density | 40 | Quality and depth of extractable text content |
+| Robots.txt AI bot access | 25 | Whether AI crawlers are allowed in robots.txt |
+| Schema.org JSON-LD | 25 | Structured data markup (Product, Article, FAQ, etc.) |
+| llms.txt presence | 10 | Whether a /llms.txt file exists for LLM guidance |
+
+### Scoring rationale (2026-02-18)
+
+The weights reflect how AI search engines (ChatGPT, Perplexity, Claude) actually consume web content:
+
+- **Content density (40 pts)** is weighted highest because it's what LLMs extract and cite when answering questions. Rich, well-structured content with headings and lists gives AI better material to work with.
+- **Robots.txt (25 pts)** is the gatekeeper — if a bot is blocked, it literally cannot crawl. It's critical but largely binary (either you're blocking or you're not).
+- **Schema.org (25 pts)** provides structured "cheat sheets" that help AI understand entities (products, articles, organizations). Valuable but not required for citation.
+- **llms.txt (10 pts)** is an emerging standard. No major AI search engine heavily weights it yet, but it signals forward-thinking AI readiness.
+
+## AI Bots Checked
+
+AEO-CLI checks access rules for these AI crawlers:
+
+- GPTBot
+- ChatGPT-User
+- Google-Extended
+- ClaudeBot
+- PerplexityBot
+- Amazonbot
+- OAI-SearchBot
+
+## Development
+
+```bash
+# Install with dev dependencies
+pip install -e ".[dev]"
+
+# Run tests
+pytest
+
+# Lint
+ruff check src/ tests/
+```
+
+## License
+
+MIT

aeo_cli-0.2.1/README.md

@@ -0,0 +1,261 @@
+# AEO-CLI
+
+[![Tests](https://github.com/hanselhansel/aeo-cli/actions/workflows/test.yml/badge.svg)](https://github.com/hanselhansel/aeo-cli/actions/workflows/test.yml)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://www.python.org/downloads/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE)
+[![PyPI version](https://img.shields.io/pypi/v/aeo-cli.svg)](https://pypi.org/project/aeo-cli/)
+
+**Audit any URL for AI crawler readiness. Get a 0-100 AEO score.**
+
+## What is AEO?
+
+Agentic Engine Optimization (AEO) is the practice of making your website discoverable and accessible to AI agents and LLM-powered search engines. As AI crawlers like GPTBot, ClaudeBot, and PerplexityBot become major traffic sources, AEO ensures your content is structured, permitted, and optimized for these systems.
+
+AEO-CLI checks how well a URL is prepared for AI consumption and returns a structured score.
+
+## Features
+
+- **Robots.txt AI bot access** — checks whether major AI crawlers are allowed or blocked
+- **llms.txt presence** — detects the emerging standard for LLM-specific site instructions
+- **Schema.org JSON-LD** — extracts and evaluates structured data markup
+- **Content density** — measures useful content vs. boilerplate via markdown conversion
+- **Rich CLI output** — formatted tables and scores via Rich
+- **JSON output** — machine-readable results for pipelines
+- **MCP server** — expose the audit as a tool for AI agents via FastMCP
+- **AEO Compiler** — LLM-powered `llms.txt` and `schema.jsonld` generation
+- **CI/CD integration** — `--fail-under` threshold, `--fail-on-blocked-bots`, GitHub Step Summary
+- **GitHub Action** — composite action for CI pipelines
+
+## Installation
+
+```bash
+pip install aeo-cli
+```
+
+AEO-CLI uses a headless browser for content extraction. After installing, run:
+
+```bash
+crawl4ai-setup
+```
+
+### Development install
+
+```bash
+git clone https://github.com/your-org/aeo-cli.git
+cd aeo-cli
+pip install -e ".[dev]"
+crawl4ai-setup
+```
+
+## Quick Start
+
+```bash
+aeo-cli audit example.com
+```
+
+This runs a full audit and prints a Rich-formatted report with your AEO score.
+
+## CLI Usage
+
+### Single Page Audit
+
+Audit only the specified URL (skip multi-page discovery):
+
+```bash
+aeo-cli audit example.com --single
+```
+
+### Multi-Page Site Audit (default)
+
+Discover pages via sitemap/spider and audit up to 10 pages:
+
+```bash
+aeo-cli audit example.com
+```
+
+### Limit Pages
+
+```bash
+aeo-cli audit example.com --max-pages 5
+```
+
+### JSON Output
+
+Get structured JSON for CI pipelines, dashboards, or scripting:
+
+```bash
+aeo-cli audit example.com --json
+```
+
+### CSV / Markdown Output
+
+```bash
+aeo-cli audit example.com --format csv
+aeo-cli audit example.com --format markdown
+```
+
+### Verbose Mode
+
+Show detailed per-pillar breakdown with scoring explanations:
+
+```bash
+aeo-cli audit example.com --single --verbose
+```
+
+### CI Mode
+
+Fail the build if the AEO score is below a threshold:
+
+```bash
+aeo-cli audit example.com --fail-under 60
+```
+
+Fail if any AI bot is blocked:
+
+```bash
+aeo-cli audit example.com --fail-on-blocked-bots
+```
+
+Exit codes: 0 = pass, 1 = score below threshold, 2 = bots blocked.
+
+When running in GitHub Actions, a markdown summary is automatically written to `$GITHUB_STEP_SUMMARY`.
+
+### Quiet Mode
+
+Suppress output, exit code 0 if score >= 50, 1 otherwise:
+
+```bash
+aeo-cli audit example.com --quiet
+```
+
+Use `--fail-under` with `--quiet` to override the default threshold:
+
+```bash
+aeo-cli audit example.com --quiet --fail-under 70
+```
+
+### Start MCP server
+
+```bash
+aeo-cli mcp
+```
+
+Launches a FastMCP stdio server exposing the audit as a tool for AI agents.
+
+## MCP Integration
+
+To use AEO-CLI as a tool in Claude Desktop, add this to your Claude Desktop config (`claude_desktop_config.json`):
+
+```json
+{
+  "mcpServers": {
+    "aeo-cli": {
+      "command": "aeo-cli",
+      "args": ["mcp"]
+    }
+  }
+}
+```
+
+Once configured, Claude can call the `audit_url` tool directly to check any URL's AEO readiness.
+
+## AEO Compiler (Generate)
+
+Generate `llms.txt` and `schema.jsonld` files from any URL using LLM analysis:
+
+```bash
+pip install aeo-cli[generate]
+aeo-cli generate example.com
+```
+
+This crawls the URL, sends the content to an LLM, and writes optimized files to `./aeo-output/`.
+
+### BYOK (Bring Your Own Key)
+
+The generate command auto-detects your LLM provider from environment variables:
+
+| Priority | Env Variable | Model Used |
+|----------|-------------|------------|
+| 1 | `OPENAI_API_KEY` | gpt-4o-mini |
+| 2 | `ANTHROPIC_API_KEY` | claude-3-haiku-20240307 |
+| 3 | Ollama running locally | ollama/llama3.2 |
+
+Override with `--model`:
+
+```bash
+aeo-cli generate example.com --model gpt-4o
+```
+
+### Industry Profiles
+
+Tailor the output with `--profile`:
+
+```bash
+aeo-cli generate example.com --profile saas
+aeo-cli generate example.com --profile ecommerce
+```
+
+Available: `generic`, `cpg`, `saas`, `ecommerce`, `blog`.
+
+## GitHub Action
+
+Use AEO-CLI in your CI pipeline:
+
+```yaml
+- name: Run AEO Audit
+  uses: hanselhansel/aeo-cli@main
+  with:
+    url: 'https://your-site.com'
+    fail-under: '60'
+```
+
+The action sets up Python, installs aeo-cli, and runs the audit. Outputs `score` and `report-json` for downstream steps. See [docs/ci-integration.md](docs/ci-integration.md) for full documentation.
+
+## Score Breakdown
+
+AEO-CLI returns a score from 0 to 100, composed of four pillars:
+
+| Pillar | Max Points | What it measures |
+|---|---|---|
+| Content density | 40 | Quality and depth of extractable text content |
+| Robots.txt AI bot access | 25 | Whether AI crawlers are allowed in robots.txt |
+| Schema.org JSON-LD | 25 | Structured data markup (Product, Article, FAQ, etc.) |
+| llms.txt presence | 10 | Whether a /llms.txt file exists for LLM guidance |
+
+### Scoring rationale (2026-02-18)
+
+The weights reflect how AI search engines (ChatGPT, Perplexity, Claude) actually consume web content:
+
+- **Content density (40 pts)** is weighted highest because it's what LLMs extract and cite when answering questions. Rich, well-structured content with headings and lists gives AI better material to work with.
+- **Robots.txt (25 pts)** is the gatekeeper — if a bot is blocked, it literally cannot crawl. It's critical but largely binary (either you're blocking or you're not).
+- **Schema.org (25 pts)** provides structured "cheat sheets" that help AI understand entities (products, articles, organizations). Valuable but not required for citation.
+- **llms.txt (10 pts)** is an emerging standard. No major AI search engine heavily weights it yet, but it signals forward-thinking AI readiness.
+
+## AI Bots Checked
+
+AEO-CLI checks access rules for these AI crawlers:
+
+- GPTBot
+- ChatGPT-User
+- Google-Extended
+- ClaudeBot
+- PerplexityBot
+- Amazonbot
+- OAI-SearchBot
+
+## Development
+
+```bash
+# Install with dev dependencies
+pip install -e ".[dev]"
+
+# Run tests
+pytest
+
+# Lint
+ruff check src/ tests/
+```
+
+## License
+
+MIT

aeo_cli-0.2.1/pyproject.toml

@@ -0,0 +1,96 @@
+[build-system]
+requires = ["setuptools>=68.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "aeo-cli"
+version = "0.2.1"
+description = "Agentic Engine Optimization CLI — audit URLs for AI crawler readiness"
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.10"
+authors = [
+    { name = "Hansel Wahjono" },
+]
+keywords = ["aeo", "seo", "ai", "llm", "crawler", "audit", "robots-txt", "schema-org"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Internet :: WWW/HTTP",
+    "Topic :: Software Development :: Quality Assurance",
+    "Typing :: Typed",
+]
+
+dependencies = [
+    "typer>=0.9",
+    "rich>=13.0",
+    "httpx>=0.27",
+    "beautifulsoup4>=4.12",
+    "pydantic>=2.0",
+    "crawl4ai>=0.4",
+    "fastmcp>=2.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/hanselhansel/aeo-cli"
+Repository = "https://github.com/hanselhansel/aeo-cli"
+Issues = "https://github.com/hanselhansel/aeo-cli/issues"
+
+[project.optional-dependencies]
+generate = [
+    "litellm>=1.40",
+]
+dev = [
+    "pytest>=8.0",
+    "pytest-asyncio>=0.23",
+    "pytest-cov>=5.0",
+    "mypy>=1.10",
+    "ruff>=0.4",
+    "litellm>=1.40",
+]
+
+[project.scripts]
+aeo-cli = "aeo_cli.main:app"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "W"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = ["tests"]
+addopts = "--cov=aeo_cli --cov-report=term-missing --cov-report=html"
+
+[tool.coverage.run]
+source = ["src/aeo_cli"]
+omit = ["src/aeo_cli/__init__.py"]
+
+[tool.coverage.report]
+exclude_lines = [
+    "pragma: no cover",
+    "if __name__ == .__main__.",
+    "if TYPE_CHECKING:",
+]
+
+[tool.mypy]
+python_version = "3.10"
+warn_return_any = true
+warn_unused_configs = true
+disallow_untyped_defs = false
+check_untyped_defs = true
+plugins = ["pydantic.mypy"]
+
+[[tool.mypy.overrides]]
+module = ["crawl4ai.*", "bs4.*", "litellm.*"]
+ignore_missing_imports = true

aeo_cli-0.2.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

aeo_cli-0.2.1/src/aeo_cli/__init__.py

@@ -0,0 +1,8 @@
+"""AEO-CLI: Agentic Engine Optimization CLI tool."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("aeo-cli")
+except PackageNotFoundError:
+    __version__ = "0.0.0"