agentfriendly 0.1.0__py3-none-any.whl

agentfriendly/__init__.py

@@ -0,0 +1,82 @@
+"""
+agentfriendly — Python SDK
+
+Make your Python web application agent-friendly.
+
+Supports FastAPI, Django, Flask, and any ASGI/WSGI framework.
+
+Quick start:
+    pip install agentfriendly[fastapi]
+
+FastAPI:
+    from fastapi import FastAPI
+    from agentfriendly.adapters.fastapi import AgentFriendlyMiddleware
+    from agentfriendly import AgentFriendlyConfig
+
+    app = FastAPI()
+    app.add_middleware(AgentFriendlyMiddleware, config=AgentFriendlyConfig())
+"""
+
+from .config import (
+    AccessConfig,
+    AgentFriendlyConfig,
+    AnalyticsConfig,
+    ContentConfig,
+    DetectionConfig,
+    DiscoveryConfig,
+    MonetizationConfig,
+    MultiTenancyConfig,
+    PrivacyConfig,
+    ToolsConfig,
+)
+from .middleware import (
+    AgentFriendlyMiddleware,
+    ContentInstructions,
+    EarlyResponse,
+    OrchestratorResult,
+    get_agent_context,
+)
+from .multitenancy import issue_delegation_token, revoke_session, validate_delegation_token
+from .types import (
+    TIER_ORDER,
+    AgentContext,
+    AgentEntry,
+    DetectionSignal,
+    TenantContext,
+    TrustTier,
+    VerifiedIdentity,
+    meets_minimum_tier,
+)
+
+__all__ = [
+    # Config
+    "AgentFriendlyConfig",
+    "DetectionConfig",
+    "DiscoveryConfig",
+    "ContentConfig",
+    "AnalyticsConfig",
+    "AccessConfig",
+    "PrivacyConfig",
+    "ToolsConfig",
+    "MonetizationConfig",
+    "MultiTenancyConfig",
+    # Middleware
+    "AgentFriendlyMiddleware",
+    "get_agent_context",
+    "OrchestratorResult",
+    "EarlyResponse",
+    "ContentInstructions",
+    # Types
+    "TrustTier",
+    "DetectionSignal",
+    "AgentContext",
+    "AgentEntry",
+    "VerifiedIdentity",
+    "TenantContext",
+    "TIER_ORDER",
+    "meets_minimum_tier",
+    # Multi-tenancy
+    "issue_delegation_token",
+    "validate_delegation_token",
+    "revoke_session",
+]

agentfriendly/access/__init__.py

@@ -0,0 +1,18 @@
+"""Layer 4 — Access Control"""
+
+from .policy_engine import (
+    PolicyDecision,
+    PolicyResult,
+    evaluate_policy,
+    generate_robots_txt_ai_section,
+)
+from .rate_limiter import InMemoryRateLimiter, get_rate_limit_key
+
+__all__ = [
+    "evaluate_policy",
+    "PolicyDecision",
+    "PolicyResult",
+    "generate_robots_txt_ai_section",
+    "InMemoryRateLimiter",
+    "get_rate_limit_key",
+]

agentfriendly/access/policy_engine.py

@@ -0,0 +1,137 @@
+"""
+Layer 4 — Access Control Policy Engine
+
+Mirrors packages/core/src/access/policy-engine.ts
+"""
+
+from __future__ import annotations
+
+import fnmatch
+from dataclasses import dataclass
+from typing import Literal
+
+from ..config import AccessConfig, AgentTypePolicy
+from ..types import AgentContext
+
+PolicyDecision = Literal["allow", "deny", "rate-limit"]
+
+CATEGORY_UA_REPRESENTATIVES: dict[str, list[str]] = {
+    "training-crawler": [
+        "GPTBot", "ClaudeBot", "Google-Extended", "CCBot",
+        "Bytespider", "anthropic-ai", "Meta-ExternalAgent",
+        "Amazonbot", "Applebot-Extended", "cohere-ai", "AI2Bot",
+    ],
+    "search-bot": [
+        "OAI-SearchBot", "ChatGPT-User", "PerplexityBot",
+        "YouBot", "DuckAssistBot",
+    ],
+    "interactive-agent": [
+        "GoogleAgent-URLContext", "Claude-Web", "Claude-SearchBot",
+    ],
+}
+
+
+@dataclass(frozen=True)
+class PolicyResult:
+    decision: PolicyDecision
+    reason: str
+    status_code: int | None
+
+
+_ALLOW = PolicyResult(decision="allow", reason="No matching deny rules", status_code=None)
+
+
+def _deny(reason: str) -> PolicyResult:
+    return PolicyResult(decision="deny", reason=reason, status_code=403)
+
+
+def _apply_agent_policy(
+    policy: AgentTypePolicy,
+    path: str,
+    allow_patterns: list[str],
+) -> PolicyResult | None:
+    """Apply a named policy. Returns None to continue evaluation, or a final result."""
+    if policy == "deny-all":
+        return _deny("Agent type policy: deny-all")
+    if policy == "allow-all":
+        return None
+    if policy == "allow-public":
+        if allow_patterns and not any(fnmatch.fnmatch(path, p) for p in allow_patterns):
+            return _deny("Agent type policy: allow-public — path not in allow list")
+        return None
+    return None
+
+
+def evaluate_policy(context: AgentContext, config: AccessConfig) -> PolicyResult:
+    """Evaluate the access policy for an agent request."""
+    if not context.is_agent:
+        return _ALLOW
+
+    path = context.path
+    allow_patterns = config.allow or []
+
+    # 1. Per-operator overrides
+    if config.operators and context.matched_agent:
+        op_policy = config.operators.get(context.matched_agent.operator)
+        if op_policy:
+            result = _apply_agent_policy(op_policy, path, allow_patterns)
+            if result:
+                return result
+
+    # 2. Per-category overrides
+    if config.agent_types and context.agent_category:
+        cat_policy = config.agent_types.get(context.agent_category)
+        if cat_policy:
+            result = _apply_agent_policy(cat_policy, path, allow_patterns)
+            if result:
+                return result
+
+    # 3. Suspected-agent policy
+    if context.tier == "suspected-agent":
+        sus_policy = config.agent_types.get("suspected-agent") if config.agent_types else None
+        if sus_policy:
+            result = _apply_agent_policy(sus_policy, path, allow_patterns)
+            if result:
+                return result
+
+    # 4. Route-level deny rules
+    deny_patterns = config.deny or []
+    if deny_patterns and any(fnmatch.fnmatch(path, p) for p in deny_patterns):
+        if allow_patterns and any(fnmatch.fnmatch(path, p) for p in allow_patterns):
+            return _ALLOW
+        return _deny(f'Path "{path}" matches deny pattern')
+
+    return _ALLOW
+
+
+def generate_robots_txt_ai_section(config: AccessConfig) -> str:
+    """Generate the AI/agent section of robots.txt from config."""
+    lines = [
+        "# AI Agent Access Control — generated by agentfriendly",
+        "# Modify these rules via access.agent_types in AgentFriendlyConfig",
+        "",
+    ]
+
+    for category, policy in (config.agent_types or {}).items():
+        reps = CATEGORY_UA_REPRESENTATIVES.get(str(category), [])
+        if not reps:
+            continue
+
+        lines.append(f"# {category} — policy: {policy}")
+        for ua in reps:
+            lines.append(f"User-agent: {ua}")
+
+        if policy == "deny-all":
+            lines.append("Disallow: /")
+        elif policy == "allow-public":
+            denied = [p for p in (config.deny or []) if "*" not in p]
+            for path in denied:
+                lines.append(f"Disallow: {path}")
+            if not denied:
+                lines.append("Allow: /")
+        else:
+            lines.append("Allow: /")
+
+        lines.append("")
+
+    return "\n".join(lines)

agentfriendly/access/rate_limiter.py

@@ -0,0 +1,66 @@
+"""Layer 4 — In-Memory Rate Limiter"""
+
+from __future__ import annotations
+
+import time
+from collections import defaultdict
+
+from ..types import AgentContext
+
+
+class InMemoryRateLimiter:
+    """
+    Simple sliding window rate limiter.
+    Thread-safe for single-process deployments.
+    """
+
+    def __init__(self, max_requests: int, window_seconds: int = 60) -> None:
+        self.max_requests = max_requests
+        self.window_seconds = window_seconds
+        self._windows: dict[str, list[float]] = defaultdict(list)
+
+    def check(self, key: str) -> bool:
+        """Returns True if the request is within the limit, False if rate-limited."""
+        now = time.monotonic()
+        window_start = now - self.window_seconds
+        timestamps = self._windows[key]
+
+        # Prune old timestamps
+        self._windows[key] = [t for t in timestamps if t > window_start]
+
+        if len(self._windows[key]) >= self.max_requests:
+            return False
+
+        self._windows[key].append(now)
+        return True
+
+    def get_count(self, key: str) -> int:
+        now = time.monotonic()
+        window_start = now - self.window_seconds
+        return sum(1 for t in self._windows[key] if t > window_start)
+
+    def clear(self) -> None:
+        self._windows.clear()
+
+
+def get_rate_limit_key(
+    context: AgentContext,
+    key_by: str = "identity",
+) -> str:
+    """Derive the rate limit key from an agent context."""
+    if key_by == "ip":
+        return (
+            context.headers.get("x-forwarded-for", "").split(",")[0].strip()
+            or context.headers.get("x-real-ip", "unknown-ip")
+        )
+    if key_by == "ua":
+        return context.user_agent or "no-ua"
+    # Default: "identity"
+    if context.verified_identity:
+        return context.verified_identity.agent_id
+    if context.matched_agent:
+        return context.matched_agent.agent_name
+    return (
+        context.headers.get("x-forwarded-for", "").split(",")[0].strip()
+        or "unknown"
+    )

agentfriendly/adapters/__init__.py

@@ -0,0 +1,3 @@
+"""
+Framework adapters for the agentfriendly Python SDK.
+"""

agentfriendly/adapters/django.py

@@ -0,0 +1,142 @@
+"""
+Django Middleware Adapter
+
+Add to MIDDLEWARE in settings.py:
+
+    MIDDLEWARE = [
+        "agentfriendly.adapters.django.AgentFriendlyMiddleware",
+        # ... other middleware
+    ]
+
+Configure in settings.py:
+
+    AGENTFRIENDLY = {
+        "detection": {"proactive_markdown": "known"},
+        "content": {"markdown": True},
+    }
+
+Access context in views:
+
+    from agentfriendly import get_agent_context
+
+    def my_view(request):
+        ctx = get_agent_context()
+        # or via request
+        ctx = getattr(request, "agent_context", None)
+"""
+
+from __future__ import annotations
+
+from django.conf import settings  # type: ignore[import-untyped]
+from django.http import HttpRequest, HttpResponse  # type: ignore[import-untyped]
+
+from ..config import AgentFriendlyConfig
+from ..content.html_to_markdown import html_to_markdown
+from ..middleware import AgentFriendlyMiddleware as CoreMiddleware
+
+
+def _load_config_from_settings() -> AgentFriendlyConfig:
+    """Load config from Django settings.AGENTFRIENDLY dict."""
+    raw = getattr(settings, "AGENTFRIENDLY", {})
+    if not isinstance(raw, dict):
+        return AgentFriendlyConfig()
+    return _dict_to_config(raw)
+
+
+def _dict_to_config(raw: dict[str, object]) -> AgentFriendlyConfig:
+    """Convert a plain settings dict to AgentFriendlyConfig."""
+    from ..config import (
+        AccessConfig,
+        AnalyticsConfig,
+        ContentConfig,
+        DetectionConfig,
+        DiscoveryConfig,
+        MonetizationConfig,
+        MultiTenancyConfig,
+        PrivacyConfig,
+        ToolsConfig,
+    )
+
+    def section(name: str, cls):  # type: ignore[no-untyped-def]
+        data = raw.get(name, {})
+        if isinstance(data, dict):
+            return cls(**{k: v for k, v in data.items() if k in cls.__dataclass_fields__})
+        return cls()
+
+    return AgentFriendlyConfig(
+        detection=section("detection", DetectionConfig),
+        discovery=section("discovery", DiscoveryConfig),
+        content=section("content", ContentConfig),
+        analytics=section("analytics", AnalyticsConfig),
+        access=section("access", AccessConfig),
+        privacy=section("privacy", PrivacyConfig),
+        tools=section("tools", ToolsConfig),
+        monetization=section("monetization", MonetizationConfig),
+        multi_tenancy=section("multi_tenancy", MultiTenancyConfig),
+        debug=bool(raw.get("debug", False)),
+    )
+
+
+class AgentFriendlyMiddleware:
+    """Django middleware class (get_response pattern)."""
+
+    def __init__(self, get_response):  # type: ignore[no-untyped-def]
+        self.get_response = get_response
+        config = _load_config_from_settings()
+        self._sdk = CoreMiddleware(config)
+
+    def __call__(self, request: HttpRequest):  # type: ignore[no-untyped-def]
+        import asyncio
+
+        # Run the async SDK process in the sync Django context
+        loop = asyncio.new_event_loop()
+        try:
+            result = loop.run_until_complete(
+                self._sdk.process(
+                    method=request.method,
+                    path=request.path,
+                    headers={k.lower(): v for k, v in request.META.items() if k.startswith("HTTP_")},
+                    url=request.build_absolute_uri(),
+                )
+            )
+        finally:
+            loop.close()
+
+        # Attach context to the request
+        request.agent_context = result.context  # type: ignore[attr-defined]
+
+        # Serve early responses
+        if result.early_response:
+            er = result.early_response
+            response = HttpResponse(
+                content=er.body,
+                status=er.status,
+                content_type=er.content_type,
+            )
+            for key, value in er.headers.items():
+                response[key] = value
+            return response
+
+        # Let Django handle the request
+        response = self.get_response(request)
+
+        # Inject agent headers
+        for key, value in result.content_instructions.agent_headers.items():
+            response[key] = value
+
+        # Convert HTML→Markdown for agent requests
+        if result.content_instructions.convert_to_markdown:
+            ct = response.get("Content-Type", "")
+            if "text/html" in ct:
+                body = response.content.decode("utf-8", errors="replace")
+                md_result = html_to_markdown(
+                    body,
+                    request.build_absolute_uri(),
+                    result.content_instructions.additional_strip_selectors,
+                )
+                response.content = md_result.markdown.encode("utf-8")
+                response["Content-Type"] = "text/markdown; charset=utf-8"
+                response["x-markdown-tokens"] = str(md_result.estimated_tokens)
+                del response["Content-Length"]
+
+        return response

agentfriendly/adapters/fastapi.py

@@ -0,0 +1,113 @@
+"""
+FastAPI / Starlette Adapter
+
+Add as ASGI middleware:
+
+    from agentfriendly.adapters.fastapi import AgentFriendlyMiddleware as AFMiddleware
+    from agentfriendly import AgentFriendlyConfig
+
+    app.add_middleware(
+        AFMiddleware,
+        config=AgentFriendlyConfig(
+            detection=DetectionConfig(proactive_markdown="known"),
+        ),
+    )
+
+Access context in route handlers:
+
+    from agentfriendly import get_agent_context
+
+    @app.get("/docs")
+    async def docs(request: Request):
+        ctx = get_agent_context()
+        return {"is_agent": ctx.is_agent if ctx else False}
+"""
+
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable
+
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.requests import Request
+from starlette.responses import Response
+
+from ..config import AgentFriendlyConfig
+from ..content.html_to_markdown import html_to_markdown
+from ..middleware import AgentFriendlyMiddleware as CoreMiddleware
+
+
+class AgentFriendlyMiddleware(BaseHTTPMiddleware):
+    """
+    Starlette/FastAPI ASGI middleware.
+    Compatible with FastAPI, Starlette, and any ASGI framework.
+    """
+
+    def __init__(self, app: object, config: AgentFriendlyConfig | None = None) -> None:
+        super().__init__(app)  # type: ignore[arg-type]
+        self._sdk = CoreMiddleware(config)
+
+    async def dispatch(
+        self,
+        request: Request,
+        call_next: Callable[[Request], Awaitable[Response]],
+    ) -> Response:
+        # Build header dict (lowercased)
+        headers = {k.lower(): v for k, v in request.headers.items()}
+        path = request.url.path
+        url = str(request.url)
+
+        result = await self._sdk.process(
+            method=request.method,
+            path=path,
+            headers=headers,
+            url=url,
+        )
+
+        # Attach context to request state for downstream access
+        request.state.agent_context = result.context
+
+        # Serve early responses directly
+        if result.early_response:
+            er = result.early_response
+            return Response(
+                content=er.body,
+                status_code=er.status,
+                headers=er.headers,
+                media_type=er.content_type,
+            )
+
+        # Let the route handler produce a response
+        response = await call_next(request)
+
+        # Inject agent headers
+        for key, value in result.content_instructions.agent_headers.items():
+            response.headers[key] = value
+
+        # Convert HTML→Markdown for agent requests
+        if result.content_instructions.convert_to_markdown:
+            content_type = response.headers.get("content-type", "")
+            if "text/html" in content_type:
+                body_bytes = b""
+                async for chunk in response.body_iterator:  # type: ignore[attr-defined]
+                    body_bytes += chunk if isinstance(chunk, bytes) else chunk.encode()
+                html_body = body_bytes.decode("utf-8", errors="replace")
+
+                md_result = html_to_markdown(
+                    html_body,
+                    url,
+                    result.content_instructions.additional_strip_selectors,
+                )
+
+                new_headers = dict(response.headers)
+                new_headers["content-type"] = "text/markdown; charset=utf-8"
+                new_headers["x-markdown-tokens"] = str(md_result.estimated_tokens)
+                new_headers.pop("content-length", None)
+
+                return Response(
+                    content=md_result.markdown.encode("utf-8"),
+                    status_code=response.status_code,
+                    headers=new_headers,
+                    media_type="text/markdown",
+                )
+
+        return response

agentfriendly/adapters/flask.py

@@ -0,0 +1,99 @@
+"""
+Flask Adapter
+
+Usage:
+
+    from flask import Flask
+    from agentfriendly.adapters.flask import init_app
+    from agentfriendly import AgentFriendlyConfig
+
+    app = Flask(__name__)
+    init_app(
+        app,
+        config=AgentFriendlyConfig(
+            detection=DetectionConfig(proactive_markdown="known"),
+        ),
+    )
+
+Access context in views:
+
+    from agentfriendly import get_agent_context
+
+    @app.route("/docs")
+    def docs():
+        ctx = get_agent_context()
+        from flask import g
+        ctx = g.agent_context  # also available on Flask's g object
+"""
+
+from __future__ import annotations
+
+import asyncio
+
+from flask import Flask, Response, g, make_response  # type: ignore[import-untyped]
+from flask import request as flask_request
+
+from ..config import AgentFriendlyConfig
+from ..content.html_to_markdown import html_to_markdown
+from ..middleware import AgentFriendlyMiddleware as CoreMiddleware
+
+
+def init_app(app: Flask, config: AgentFriendlyConfig | None = None) -> None:
+    """Register AgentFriendly before/after request hooks on a Flask app."""
+    sdk = CoreMiddleware(config)
+
+    @app.before_request
+    def before_request() -> Response | None:  # type: ignore[return]
+        headers = {k.lower(): v for k, v in flask_request.headers.items()}
+        loop = asyncio.new_event_loop()
+        try:
+            result = loop.run_until_complete(
+                sdk.process(
+                    method=flask_request.method,
+                    path=flask_request.path,
+                    headers=headers,
+                    url=flask_request.url,
+                )
+            )
+        finally:
+            loop.close()
+
+        g.agent_result = result
+        g.agent_context = result.context
+
+        # Serve early responses before the route handler
+        if result.early_response:
+            er = result.early_response
+            resp = make_response(er.body, er.status)
+            resp.content_type = er.content_type
+            for key, value in er.headers.items():
+                resp.headers[key] = value
+            return resp
+
+        return None
+
+    @app.after_request
+    def after_request(response: Response) -> Response:
+        result = getattr(g, "agent_result", None)
+        if not result:
+            return response
+
+        # Inject agent headers
+        for key, value in result.content_instructions.agent_headers.items():
+            response.headers[key] = value
+
+        # Convert HTML→Markdown for agent requests
+        if result.content_instructions.convert_to_markdown:
+            ct = response.content_type or ""
+            if "text/html" in ct:
+                body = response.get_data(as_text=True)
+                md_result = html_to_markdown(
+                    body,
+                    flask_request.url,
+                    result.content_instructions.additional_strip_selectors,
+                )
+                response.set_data(md_result.markdown)
+                response.content_type = "text/markdown; charset=utf-8"
+                response.headers["x-markdown-tokens"] = str(md_result.estimated_tokens)
+
+        return response