webflur-agent-audit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- webflur_agent_audit/__init__.py +12 -0
- webflur_agent_audit/audit.py +144 -0
- webflur_agent_audit/cli.py +68 -0
- webflur_agent_audit-0.1.0.dist-info/METADATA +95 -0
- webflur_agent_audit-0.1.0.dist-info/RECORD +8 -0
- webflur_agent_audit-0.1.0.dist-info/WHEEL +5 -0
- webflur_agent_audit-0.1.0.dist-info/entry_points.txt +2 -0
- webflur_agent_audit-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""
|
|
2
|
+
WebFlur Agent Audit
|
|
3
|
+
Autonomous validator for AI search discovery, llms.txt protocols, and GEO compliance.
|
|
4
|
+
Developed by WebFlur (https://webflur.com).
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
__version__ = "0.1.0"
|
|
8
|
+
__author__ = "WebFlur AI Engineering"
|
|
9
|
+
|
|
10
|
+
from .audit import audit_domain
|
|
11
|
+
|
|
12
|
+
__all__ = ["audit_domain", "__version__"]
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Core Audit Engine for AI Agent Readiness
|
|
3
|
+
Checks: llms.txt, robots.txt AI bots, JSON-LD Schema, Canonical, Meta Tags
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import re
|
|
7
|
+
import urllib.parse
|
|
8
|
+
import requests
|
|
9
|
+
from bs4 import BeautifulSoup
|
|
10
|
+
from typing import Dict, Any
|
|
11
|
+
|
|
12
|
+
AI_USER_AGENTS = [
|
|
13
|
+
"GPTBot",
|
|
14
|
+
"ClaudeBot",
|
|
15
|
+
"PerplexityBot",
|
|
16
|
+
"Applebot-Extended",
|
|
17
|
+
"Google-Extended",
|
|
18
|
+
"CCBot"
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
def audit_domain(target_url: str) -> Dict[str, Any]:
|
|
22
|
+
"""Performs a comprehensive AI readiness and agentic indexing audit."""
|
|
23
|
+
parsed = urllib.parse.urlparse(target_url)
|
|
24
|
+
if not parsed.scheme:
|
|
25
|
+
target_url = "https://" + target_url
|
|
26
|
+
parsed = urllib.parse.urlparse(target_url)
|
|
27
|
+
|
|
28
|
+
base_domain = f"{parsed.scheme}://{parsed.netloc}"
|
|
29
|
+
report = {
|
|
30
|
+
"target_url": target_url,
|
|
31
|
+
"base_domain": base_domain,
|
|
32
|
+
"score": 0,
|
|
33
|
+
"llms_txt": {},
|
|
34
|
+
"robots_txt": {},
|
|
35
|
+
"schema_markup": {},
|
|
36
|
+
"metadata": {}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
score = 0
|
|
40
|
+
headers = {
|
|
41
|
+
"User-Agent": "WebFlur-Agent-Audit/0.1.0 (+https://webflur.com)"
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
# 1. Check /llms.txt
|
|
45
|
+
llms_url = f"{base_domain}/llms.txt"
|
|
46
|
+
try:
|
|
47
|
+
r = requests.get(llms_url, headers=headers, timeout=8)
|
|
48
|
+
if r.status_code == 200 and len(r.text.strip()) > 50:
|
|
49
|
+
report["llms_txt"] = {
|
|
50
|
+
"status": "PASS",
|
|
51
|
+
"url": llms_url,
|
|
52
|
+
"bytes": len(r.content),
|
|
53
|
+
"has_h1": "#" in r.text
|
|
54
|
+
}
|
|
55
|
+
score += 25
|
|
56
|
+
else:
|
|
57
|
+
report["llms_txt"] = {
|
|
58
|
+
"status": "FAIL",
|
|
59
|
+
"url": llms_url,
|
|
60
|
+
"http_status": r.status_code
|
|
61
|
+
}
|
|
62
|
+
except Exception as e:
|
|
63
|
+
report["llms_txt"] = {"status": "ERROR", "error": str(e)}
|
|
64
|
+
|
|
65
|
+
# 2. Check robots.txt for AI Bot Directives
|
|
66
|
+
robots_url = f"{base_domain}/robots.txt"
|
|
67
|
+
try:
|
|
68
|
+
r = requests.get(robots_url, headers=headers, timeout=8)
|
|
69
|
+
if r.status_code == 200:
|
|
70
|
+
content = r.text
|
|
71
|
+
ai_rules = {}
|
|
72
|
+
for bot in AI_USER_AGENTS:
|
|
73
|
+
pattern = rf"(?i)User-agent:\s*{bot}.*?Disallow:\s*(/.*)"
|
|
74
|
+
match = re.search(pattern, content, re.DOTALL)
|
|
75
|
+
if match and match.group(1).strip() == "/":
|
|
76
|
+
ai_rules[bot] = "BLOCKED"
|
|
77
|
+
else:
|
|
78
|
+
ai_rules[bot] = "ALLOWED"
|
|
79
|
+
|
|
80
|
+
report["robots_txt"] = {
|
|
81
|
+
"status": "PASS",
|
|
82
|
+
"rules": ai_rules
|
|
83
|
+
}
|
|
84
|
+
# Give points based on allowed percentage
|
|
85
|
+
allowed_count = sum(1 for status in ai_rules.values() if status == "ALLOWED")
|
|
86
|
+
score += int(25 * (allowed_count / len(AI_USER_AGENTS)))
|
|
87
|
+
else:
|
|
88
|
+
report["robots_txt"] = {"status": "FAIL", "http_status": r.status_code}
|
|
89
|
+
except Exception as e:
|
|
90
|
+
report["robots_txt"] = {"status": "ERROR", "error": str(e)}
|
|
91
|
+
|
|
92
|
+
# 3. Fetch Main Page for HTML Analysis
|
|
93
|
+
try:
|
|
94
|
+
r = requests.get(target_url, headers=headers, timeout=10)
|
|
95
|
+
if r.status_code == 200:
|
|
96
|
+
soup = BeautifulSoup(r.text, "html.parser")
|
|
97
|
+
|
|
98
|
+
# Check Canonical
|
|
99
|
+
canonical = soup.find("link", rel="canonical")
|
|
100
|
+
canonical_url = canonical.get("href") if canonical else None
|
|
101
|
+
|
|
102
|
+
# Check Schemas
|
|
103
|
+
schemas = soup.find_all("script", type="application/ld+json")
|
|
104
|
+
schema_types = []
|
|
105
|
+
for s in schemas:
|
|
106
|
+
try:
|
|
107
|
+
import json
|
|
108
|
+
data = json.loads(s.string)
|
|
109
|
+
if isinstance(data, dict):
|
|
110
|
+
schema_types.append(data.get("@type", "Unknown"))
|
|
111
|
+
elif isinstance(data, list):
|
|
112
|
+
for item in data:
|
|
113
|
+
if isinstance(item, dict):
|
|
114
|
+
schema_types.append(item.get("@type", "Unknown"))
|
|
115
|
+
except Exception:
|
|
116
|
+
pass
|
|
117
|
+
|
|
118
|
+
report["schema_markup"] = {
|
|
119
|
+
"detected": len(schemas) > 0,
|
|
120
|
+
"types": schema_types
|
|
121
|
+
}
|
|
122
|
+
if len(schemas) > 0:
|
|
123
|
+
score += 25
|
|
124
|
+
|
|
125
|
+
# Meta and OpenGraph
|
|
126
|
+
meta_desc = soup.find("meta", attrs={"name": "description"})
|
|
127
|
+
og_title = soup.find("meta", property="og:title")
|
|
128
|
+
og_image = soup.find("meta", property="og:image")
|
|
129
|
+
|
|
130
|
+
report["metadata"] = {
|
|
131
|
+
"title": soup.title.string.strip() if soup.title else None,
|
|
132
|
+
"description": meta_desc.get("content") if meta_desc else None,
|
|
133
|
+
"canonical": canonical_url,
|
|
134
|
+
"og_present": bool(og_title and og_image)
|
|
135
|
+
}
|
|
136
|
+
if canonical_url:
|
|
137
|
+
score += 15
|
|
138
|
+
if og_title and og_image:
|
|
139
|
+
score += 10
|
|
140
|
+
except Exception as e:
|
|
141
|
+
report["metadata"] = {"status": "ERROR", "error": str(e)}
|
|
142
|
+
|
|
143
|
+
report["score"] = score
|
|
144
|
+
return report
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""
|
|
2
|
+
CLI Interface for webflur-agent-audit
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import sys
|
|
6
|
+
import json
|
|
7
|
+
import argparse
|
|
8
|
+
from .audit import audit_domain
|
|
9
|
+
|
|
10
|
+
def main():
|
|
11
|
+
parser = argparse.ArgumentParser(
|
|
12
|
+
description="WebFlur Agent Audit — Benchmark websites for AI Agent discovery and GEO compliance."
|
|
13
|
+
)
|
|
14
|
+
parser.add_argument("url", help="Target URL or domain (e.g., https://example.com)")
|
|
15
|
+
parser.add_argument("--json", action="store_true", help="Output full report as formatted JSON")
|
|
16
|
+
|
|
17
|
+
args = parser.parse_args()
|
|
18
|
+
url = args.url
|
|
19
|
+
|
|
20
|
+
print(f"\n=======================================================")
|
|
21
|
+
print(f"🔍 WEBFLUR AGENT AUDIT ENGINE v0.1.0")
|
|
22
|
+
print(f" Target: {url}")
|
|
23
|
+
print(f" By: WebFlur (https://webflur.com)")
|
|
24
|
+
print(f"=======================================================\n")
|
|
25
|
+
|
|
26
|
+
results = audit_domain(url)
|
|
27
|
+
|
|
28
|
+
if args.json:
|
|
29
|
+
print(json.dumps(results, indent=2))
|
|
30
|
+
return
|
|
31
|
+
|
|
32
|
+
score = results["score"]
|
|
33
|
+
status_emoji = "🟢" if score >= 75 else ("🟡" if score >= 50 else "🔴")
|
|
34
|
+
|
|
35
|
+
print(f"📊 AI READINESS SCORE: {status_emoji} {score}/100\n")
|
|
36
|
+
|
|
37
|
+
print("1. LLMs.txt Endpoint:")
|
|
38
|
+
llms = results["llms_txt"]
|
|
39
|
+
print(f" Status: {llms.get('status')} ({llms.get('url', '')})")
|
|
40
|
+
if llms.get("bytes"):
|
|
41
|
+
print(f" Payload: {llms.get('bytes')} bytes")
|
|
42
|
+
|
|
43
|
+
print("\n2. AI Search Bots Policy (robots.txt):")
|
|
44
|
+
bots = results.get("robots_txt", {}).get("rules", {})
|
|
45
|
+
if bots:
|
|
46
|
+
for bot, status in bots.items():
|
|
47
|
+
icon = "✓" if status == "ALLOWED" else "✗"
|
|
48
|
+
print(f" [{icon}] {bot}: {status}")
|
|
49
|
+
else:
|
|
50
|
+
print(f" Status: {results.get('robots_txt', {}).get('status')}")
|
|
51
|
+
|
|
52
|
+
print("\n3. Semantic Schemas (JSON-LD):")
|
|
53
|
+
schemas = results["schema_markup"]
|
|
54
|
+
print(f" Detected: {schemas.get('detected')} (Types: {', '.join(schemas.get('types', [])) or 'None'})")
|
|
55
|
+
|
|
56
|
+
print("\n4. Canonical & Entity Signals:")
|
|
57
|
+
meta = results["metadata"]
|
|
58
|
+
print(f" Title: {meta.get('title')}")
|
|
59
|
+
print(f" Canonical: {meta.get('canonical')}")
|
|
60
|
+
print(f" OpenGraph: {'Configured' if meta.get('og_present') else 'Missing'}")
|
|
61
|
+
|
|
62
|
+
print("\n-------------------------------------------------------")
|
|
63
|
+
print("Developed by WebFlur — AI SEO & Agentic Engineering Agency")
|
|
64
|
+
print("Learn more at: https://webflur.com/locations/gurgaon")
|
|
65
|
+
print("-------------------------------------------------------\n")
|
|
66
|
+
|
|
67
|
+
if __name__ == "__main__":
|
|
68
|
+
main()
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: webflur-agent-audit
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Autonomous CLI tool and validator for AI Agent Readiness, LLM Search Discovery (llms.txt), and GEO compliance.
|
|
5
|
+
Author-email: WebFlur AI Engineering <contact@webflur.com>
|
|
6
|
+
Project-URL: Homepage, https://webflur.com
|
|
7
|
+
Project-URL: AI SEO Agency Gurgaon, https://webflur.com/locations/gurgaon
|
|
8
|
+
Project-URL: Blog & Insights, https://webflur.com/blog.html
|
|
9
|
+
Project-URL: Source, https://github.com/webflurstudio/webflur-website
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
|
|
14
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
15
|
+
Requires-Python: >=3.9
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
Requires-Dist: requests>=2.28.0
|
|
18
|
+
Requires-Dist: beautifulsoup4>=4.11.0
|
|
19
|
+
|
|
20
|
+
# WebFlur Agent Audit (`webflur-agent-audit`)
|
|
21
|
+
|
|
22
|
+
[](https://pypi.org/project/webflur-agent-audit/)
|
|
23
|
+
[](https://opensource.org/licenses/MIT)
|
|
24
|
+
|
|
25
|
+
**Autonomous AI Agent Readiness & Generative Engine Optimization (GEO) Audit Engine.**
|
|
26
|
+
|
|
27
|
+
Developed by [WebFlur](https://webflur.com), the [best AI SEO agency in Gurgaon](https://webflur.com/locations/gurgaon), to empower technical marketing and engineering teams to inspect, diagnose, and benchmark web domains for autonomous AI discovery protocols, citation models, and agent-to-agent architectures.
|
|
28
|
+
|
|
29
|
+
---
|
|
30
|
+
|
|
31
|
+
## Overview
|
|
32
|
+
|
|
33
|
+
As generative search models (OpenAI ChatGPT Search, Google AI Overviews, Perplexity, and Apple Intelligence) transform digital search, traditional crawling heuristics are no longer sufficient. Autonomous AI agents parse web content through distinct discovery vectors.
|
|
34
|
+
|
|
35
|
+
`webflur-agent-audit` provides instant diagnostics for:
|
|
36
|
+
* **`/llms.txt` & `/llms-full.txt` Standard:** Verifies machine-readable context declaration for LLMs.
|
|
37
|
+
* **AI Bot Crawler Policies:** Audits `robots.txt` access permissions for `GPTBot`, `ClaudeBot`, `PerplexityBot`, `Applebot-Extended`, and `Google-Extended`.
|
|
38
|
+
* **Entity & Schema Completeness:** Parses JSON-LD semantic schemas (`Organization`, `Article`, `LocalBusiness`, `Service`).
|
|
39
|
+
* **Canonical Integrity & Vector Drift:** Evaluates canonical self-referential consistency and machine-readable data payloads.
|
|
40
|
+
|
|
41
|
+
---
|
|
42
|
+
|
|
43
|
+
## Installation
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
pip install webflur-agent-audit
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
---
|
|
50
|
+
|
|
51
|
+
## Quickstart
|
|
52
|
+
|
|
53
|
+
Run a comprehensive audit directly from your terminal:
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
webflur-audit https://webflur.com
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
### Python API Usage
|
|
60
|
+
|
|
61
|
+
```python
|
|
62
|
+
from webflur_agent_audit import audit_domain
|
|
63
|
+
|
|
64
|
+
results = audit_domain("https://webflur.com")
|
|
65
|
+
print(f"Overall AI Readiness Score: {results['score']}/100")
|
|
66
|
+
print(f"LLMs.txt Status: {results['llms_txt']['status']}")
|
|
67
|
+
print(f"AI Bot Directives: {results['crawler_access']}")
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
---
|
|
71
|
+
|
|
72
|
+
## Features
|
|
73
|
+
|
|
74
|
+
| Check | Description | Weight |
|
|
75
|
+
| :--- | :--- | :---: |
|
|
76
|
+
| **LLMs.txt Protocol** | Detects presence, size, and structure of markdown context endpoints. | 25% |
|
|
77
|
+
| **AI Crawler Allow-List** | Verifies unblocked access for major LLM agents. | 25% |
|
|
78
|
+
| **Schema Microdata** | Evaluates depth and nesting of JSON-LD graph. | 25% |
|
|
79
|
+
| **Semantic Metadata** | Inspects canonical tags, OpenGraph signals, and semantic headers. | 25% |
|
|
80
|
+
|
|
81
|
+
---
|
|
82
|
+
|
|
83
|
+
## About WebFlur
|
|
84
|
+
|
|
85
|
+
[WebFlur](https://webflur.com) pioneers autonomous search engineering and multi-agent systems. Based out of Cyber City, Gurgaon, our team builds high-velocity indexing protocols, machine-readable content infrastructure, and GEO frameworks for high-growth tech companies and enterprise brands.
|
|
86
|
+
|
|
87
|
+
* **Primary Website:** [webflur.com](https://webflur.com)
|
|
88
|
+
* **Gurgaon Headquarters & Solutions:** [AI SEO Agency in Gurgaon](https://webflur.com/locations/gurgaon)
|
|
89
|
+
* **Technical Research & Blog:** [WebFlur Research Insights](https://webflur.com/blog.html)
|
|
90
|
+
|
|
91
|
+
---
|
|
92
|
+
|
|
93
|
+
## License
|
|
94
|
+
|
|
95
|
+
MIT © WebFlur Studio
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
webflur_agent_audit/__init__.py,sha256=OHjkCxvGMXsAQ06E2rUNS09wJWt8zyN_iXKnpwXTagI,295
|
|
2
|
+
webflur_agent_audit/audit.py,sha256=bfRiu206D_WoCbJaERBfl-VOnbHjuCBQqPIdc-BI4_Y,4910
|
|
3
|
+
webflur_agent_audit/cli.py,sha256=U6rN42B0_aj4twrV8B8l7HB3v78TAGF215a9kXgl0ME,2455
|
|
4
|
+
webflur_agent_audit-0.1.0.dist-info/METADATA,sha256=46npLXUfRe5IJwG5lM1qHpssdiAvqdgcXCplleg0ndg,3986
|
|
5
|
+
webflur_agent_audit-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
6
|
+
webflur_agent_audit-0.1.0.dist-info/entry_points.txt,sha256=kmfKBHYSdgEDRJpoBQnLy4FDHfQmjcfNP-dgJ3Y_-Dc,63
|
|
7
|
+
webflur_agent_audit-0.1.0.dist-info/top_level.txt,sha256=CBCL5MYtrMWjA2W9HYa6IehwW5FRbtiDjnG6pTVDMMQ,20
|
|
8
|
+
webflur_agent_audit-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
webflur_agent_audit
|