autoai-optimize 1.0.0__py3-none-any.whl

autoai_optimize/__init__.py

@@ -0,0 +1,18 @@
+"""autoai-optimize: automatic Schema.org / JSON-LD injection for web apps.
+
+Install once, get better Google rankings, voice-search readiness, and
+AI-agent discoverability — zero manual structured-data work.
+
+Public API:
+    from autoai_optimize import Config, optimize_html
+    from autoai_optimize.frameworks.fastapi import AutoAIMiddleware
+    from autoai_optimize.frameworks.django import AutoAIMiddleware as DjangoMiddleware
+"""
+
+from __future__ import annotations
+
+from autoai_optimize.config import Config
+from autoai_optimize.core import generate_jsonld, optimize_html
+
+__all__ = ["Config", "__version__", "generate_jsonld", "optimize_html"]
+__version__ = "0.1.1"

autoai_optimize/analyze/__init__.py

@@ -0,0 +1,17 @@
+"""Analysis layer: page-type classification, hint parsing, entity extraction."""
+
+from __future__ import annotations
+
+from autoai_optimize.analyze.classifier import Classification, PageType, classify
+from autoai_optimize.analyze.extractors import extract_article, extract_product
+from autoai_optimize.analyze.hints import PageHint, parse_hints
+
+__all__ = [
+    "Classification",
+    "PageHint",
+    "PageType",
+    "classify",
+    "extract_article",
+    "extract_product",
+    "parse_hints",
+]

autoai_optimize/analyze/classifier.py

@@ -0,0 +1,115 @@
+"""Heuristic page-type classifier.
+
+Determines whether a response represents an Article or a Product page using
+URL patterns, HTML signals, and meta tags. Confidence-scored so the core can
+fall back to silence when uncertain — a wrong schema is worse than none.
+
+Developer hints (see hints.py) always override this classification.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+
+from bs4 import BeautifulSoup
+
+ARTICLE_URL_HINTS = ("/blog/", "/post/", "/article/", "/news/", "/posts/")
+PRODUCT_URL_HINTS = ("/product/", "/products/", "/shop/", "/p/", "/item/")
+
+
+class PageType(str, Enum):
+    """Page types autoai-optimize knows how to enrich in Phase 1."""
+
+    ARTICLE = "Article"
+    PRODUCT = "Product"
+    PROFILE = "Profile"
+    UNKNOWN = "Unknown"
+
+
+@dataclass(frozen=True)
+class Classification:
+    """The result of classifying a single response."""
+
+    page_type: PageType
+    confidence: float  # 0.0–1.0
+
+
+def _score_article(url_path: str, soup: BeautifulSoup) -> float:
+    """Confidence that the page is an Article."""
+    score = 0.0
+    if any(h in url_path for h in ARTICLE_URL_HINTS):
+        score += 0.4
+    if soup.find("article"):
+        score += 0.25
+    if soup.find("time"):
+        score += 0.15
+    # Common blog/article metadata.
+    if soup.find("meta", attrs={"name": "author"}) or soup.find("meta", attrs={"property": "article:author"}):
+        score += 0.15
+    og_type = _og_type(soup)
+    if og_type == "article":
+        score += 0.3
+    return min(score, 1.0)
+
+
+def _score_product(url_path: str, soup: BeautifulSoup) -> float:
+    """Confidence that the page is a Product page."""
+    score = 0.0
+    if any(h in url_path for h in PRODUCT_URL_HINTS):
+        score += 0.4
+    # Price-like text anywhere (currency symbol followed by digits).
+    if _has_price_signal(soup):
+        score += 0.25
+    # Add-to-cart / buy controls are strong commerce signals.
+    text_lc = soup.get_text(" ") if soup.find() else ""
+    if any(k in text_lc for k in ("add to cart", "add to bag", "buy now")):
+        score += 0.2
+    # og:type=product is a strong, explicit signal.
+    og_type = _og_type(soup)
+    if og_type == "product":
+        score += 0.3
+    return min(score, 1.0)
+
+
+def _og_type(soup: BeautifulSoup) -> str | None:
+    tag = soup.find("meta", attrs={"property": "og:type"})
+    if tag and tag.get("content"):
+        return str(tag["content"]).strip().lower()
+    return None
+
+
+def _has_price_signal(soup: BeautifulSoup) -> bool:
+    """True if the page contains plausible price text (e.g. '$29.99', '€10')."""
+    import re
+
+    pattern = re.compile(r"[$€£¥₹]\s?\d|\d+[.,]\d{2}\s?(?:usd|eur|gbp)", re.IGNORECASE)
+    for _el in soup.find_all(string=pattern):
+        return True
+    # microdata/itemprop price is an even stronger, explicit signal.
+    return soup.find(itemprop="price") is not None
+
+
+def classify(url_path: str, soup: BeautifulSoup) -> Classification:
+    """Classify a parsed page. Never raises; returns UNKNOWN on no signal.
+
+    Args:
+        url_path: Path portion of the request URL (e.g. "/blog/my-post").
+        soup: Parsed HTML (BeautifulSoup).
+    """
+    art = _score_article(url_path, soup)
+    prod = _score_product(url_path, soup)
+    if art == prod == 0.0:
+        return Classification(PageType.UNKNOWN, 0.0)
+    if art >= prod:
+        return Classification(PageType.ARTICLE, art)
+    return Classification(PageType.PRODUCT, prod)
+
+def _score_profile(url_path: str, soup: BeautifulSoup) -> float:
+    score = 0.0
+    import re
+    if "/author/" in url_path or "/profile/" in url_path or "/user/" in url_path:
+        score += 0.5
+    if soup.find(class_=re.compile("profile|bio|author-card", re.I)):
+        score += 0.4
+    return min(1.0, score)

autoai_optimize/analyze/extractors.py

@@ -0,0 +1,241 @@
+"""Entity extraction per page type.
+
+Extractors read candidate fields out of the parsed HTML. They never raise on
+missing data — they simply omit the field. The schema builders decide whether
+enough was found to emit valid JSON-LD.
+
+A developer's explicit hint fields always override extracted values.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from bs4 import BeautifulSoup
+
+from autoai_optimize.analyze.hints import PageHint
+
+
+def _first_text(
+    soup: BeautifulSoup,
+    selectors: list[tuple[str, dict[str, Any]]],
+    ai_field: str | None = None,
+) -> str | None:
+    """Return the stripped text of the first matching selector, or None."""
+    for name, attrs in selectors:
+        tag = soup.find(name, attrs=attrs)
+        if tag and tag.get_text(strip=True):
+            if ai_field:
+                tag.attrs["data-ai-field"] = ai_field
+            return tag.get_text(strip=True)
+    return None
+
+
+def _meta_content(
+    soup: BeautifulSoup,
+    selectors: list[tuple[str, dict[str, Any]]],
+    ai_field: str | None = None,
+) -> str | None:
+    for name, attrs in selectors:
+        tag = soup.find(name, attrs=attrs)
+        if tag and tag.get("content"):
+            if ai_field:
+                tag.attrs["data-ai-field"] = ai_field
+            return str(tag["content"]).strip()
+    return None
+
+
+def extract_article(soup: BeautifulSoup, url: str, hint: PageHint) -> dict[str, Any]:
+    """Extract Article fields from HTML, applying hint overrides last."""
+    if soup.body:
+        soup.body.attrs["data-ai-entity"] = "article"
+    data: dict[str, Any] = {}
+
+    title = _meta_content(
+        soup,
+        [
+            ("meta", {"property": "og:title"}),
+            ("meta", {"name": "twitter:title"}),
+        ],
+    ) or _first_text(
+        soup,
+        [
+            ("h1", {}),
+            ("title", {}),
+        ],
+    )
+    if title:
+        data["headline"] = title
+
+    description = _meta_content(
+        soup,
+        [
+            ("meta", {"name": "description"}),
+            ("meta", {"property": "og:description"}),
+        ],
+    )
+    if description:
+        data["description"] = description
+
+    image = _meta_content(soup, [("meta", {"property": "og:image"})])
+    if image:
+        data["image"] = image
+
+    author = _meta_content(
+        soup,
+        [
+            ("meta", {"name": "author"}),
+            ("meta", {"property": "article:author"}),
+        ],
+    )
+    if author:
+        data["author"] = author
+
+    date_published = _meta_content(
+        soup,
+        [
+            ("meta", {"property": "article:published_time"}),
+            ("time", {"datetime": True}),
+        ],
+    )
+    if date_published:
+        data["datePublished"] = date_published
+
+    data["url"] = url
+
+    # Developer overrides win.
+    data.update(hint.fields)
+    return data
+
+
+def extract_product(soup: BeautifulSoup, url: str, hint: PageHint) -> dict[str, Any]:
+    """Extract Product fields from HTML, applying hint overrides last."""
+    if soup.body:
+        soup.body.attrs["data-ai-entity"] = "product"
+    
+    # Inject action hint for add to cart
+    import re
+    btn = soup.find(string=re.compile(r"add to cart", re.I))
+    if btn and btn.parent:
+        btn.parent.attrs["data-ai-action"] = "add_to_cart"
+        
+    data: dict[str, Any] = {}
+
+    name = _first_text(
+        soup,
+        [
+            ("h1", {}),
+            ("meta", {"itemprop": "name"}),
+        ],
+    )
+    if name:
+        data["name"] = name
+
+    description = _meta_content(
+        soup,
+        [
+            ("meta", {"name": "description"}),
+            ("meta", {"property": "og:description"}),
+        ],
+    )
+    if description:
+        data["description"] = description
+
+    image = _meta_content(soup, [("meta", {"property": "og:image"})])
+    if image:
+        data["image"] = image
+
+    # Price: prefer microdata itemprop, then scan for a currency pattern.
+    price = _extract_price(soup)
+    if price:
+        data["price"] = price
+
+    data["url"] = url
+    data.update(hint.fields)
+    return data
+
+
+def _extract_price(soup: BeautifulSoup) -> str | None:
+    """Best-effort price extraction → returns a numeric string like '29.99'.
+
+    Attempts to be locale-aware for common formats (US, EU, INR, GBP).
+    Returns None when no confident price is found.
+    """
+    import re
+
+    def normalize_num(raw: str) -> str:
+        # Remove spaces and non-breaking spaces
+        s = raw.strip().replace('\xa0', '').replace(' ', '')
+        # If both '.' and ',' present, infer decimal separator by last occurrence
+        if '.' in s and ',' in s:
+            if s.rfind('.') > s.rfind(','):
+                # '.' likely decimal separator, remove grouping commas
+                s = s.replace(',', '')
+            else:
+                # ',' likely decimal separator
+                s = s.replace('.', '').replace(',', '.')
+        elif ',' in s:
+            # If comma followed by exactly 2 digits at end, treat as decimal
+            if re.search(r",\d{1,2}$", s):
+                s = s.replace(',', '.')
+            else:
+                # otherwise remove thousands separator commas
+                s = s.replace(',', '')
+        # else only dot present or only digits
+        # Strip any non-digit/non-dot characters
+        s = re.sub(r"[^0-9.]", "", s)
+        # Trim leading/trailing dots
+        s = s.strip('.')
+        return s
+
+    # 1) microdata itemprop price
+    itemprop = soup.find(itemprop="price")
+    if itemprop:
+        raw = str(itemprop.get("content") or itemprop.get_text(strip=True))
+        m = re.search(r"\d+[.,]?\d*", raw or "")
+        if m:
+            return normalize_num(m.group(0))
+
+    # 2) Symbol-based patterns (symbol before or after)
+    symbol_pattern_pre = re.compile(r"([$€£¥₹])\s?(\d{1,3}(?:[.,]\d{3})*(?:[.,]\d{1,2})?)")
+    symbol_pattern_post = re.compile(r"(\d{1,3}(?:[.,]\d{3})*(?:[.,]\d{1,2})?)\s?([$€£¥₹])")
+    for el in soup.find_all(string=True):
+        txt = str(el)
+        m = symbol_pattern_pre.search(txt)
+        if not m:
+            m = symbol_pattern_post.search(txt)
+        if m:
+            num = m.group(2) if len(m.groups()) >= 2 else m.group(1)
+            return normalize_num(num)
+
+    # 3) Currency code suffix/prefix e.g. '1000 INR' or 'EUR 1.234,56'
+    code_pattern = re.compile(r"(\d[\d.,]*)\s?(USD|EUR|GBP|INR)", re.IGNORECASE)
+    for el in soup.find_all(string=code_pattern):
+        m = code_pattern.search(str(el))
+        if m:
+            return normalize_num(m.group(1))
+
+    # 4) Fallback: any 1-4 digit group with decimals and currency nearby
+    generic = re.compile(r"\d+[.,]\d{2}")
+    for el in soup.find_all(string=generic):
+        txt = str(el)
+        # ensure there's a currency symbol or code nearby in the element
+        if re.search(r"[$€£¥₹]|USD|EUR|GBP|INR", txt, re.IGNORECASE):
+            m = generic.search(txt)
+            if m:
+                return normalize_num(m.group(0))
+    return None
+
+def extract_profile(soup: BeautifulSoup, url: str, hint: PageHint) -> dict[str, Any]:
+    if soup.body:
+        soup.body.attrs["data-ai-entity"] = "profile"
+    data: dict[str, Any] = {}
+    name = _first_text(soup, [("h1", {}), ("h2", {})], "name")
+    if name:
+        data["name"] = name
+    job = _meta_content(soup, [("meta", {"name": "job_title"})], "jobTitle")
+    if job:
+        data["jobTitle"] = job
+    data["url"] = url
+    data.update(hint.fields)
+    return data

autoai_optimize/analyze/hints.py

@@ -0,0 +1,73 @@
+"""Developer hints — the explicit override path.
+
+When heuristics can't tell (or the developer knows better), the host app can
+pass an explicit hint. Hints always win over heuristics and bypass the
+confidence threshold.
+
+Hints can be supplied via:
+  - Framework adapter API (view attribute / decorator / response header)
+  - A context dict passed to optimize_html()
+
+All roads converge on a PageHint here.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from autoai_optimize.analyze.classifier import PageType
+
+# Canonical header a framework adapter may set on a response.
+HINT_HEADER = "X-AutoAI-Type"
+
+# Fields a developer can supply to pre-populate a schema, e.g.
+#   hints={"type": "Product", "name": "Widget", "price": "29.99"}
+_HINT_TYPE_KEY = "type"
+_HINT_FIELD_KEY = "fields"
+
+
+@dataclass(frozen=True)
+class PageHint:
+    """An explicit developer-supplied page classification / field override."""
+
+    page_type: PageType | None
+    fields: dict[str, Any]
+
+
+def _coerce_type(value: Any) -> PageType | None:
+    if value is None:
+        return None
+    if isinstance(value, PageType):
+        return value
+    name = str(value).strip().capitalize()
+    try:
+        return PageType(name)
+    except ValueError:
+        return None
+
+
+def parse_hints(raw: dict[str, Any] | None, html_content: str = "") -> PageHint:
+    """Normalize a raw hints dict (from adapter or context) into a PageHint.
+    Also parses the HTML string for explicit frontend developer comments (e.g., <!-- @ai-entity:product -->).
+    """
+    if not raw:
+        raw = {}
+        
+    import re
+    if html_content:
+        hint_match = re.search(r'<!--\s*@ai-entity:(\w+)\s*-->', html_content)
+        if hint_match and _HINT_TYPE_KEY not in raw:
+            raw[_HINT_TYPE_KEY] = hint_match.group(1).lower()
+
+    if not raw:
+        return PageHint(page_type=None, fields={})
+    raw_type = raw.get(_HINT_TYPE_KEY)
+    nested_fields = raw.get(_HINT_FIELD_KEY, {})
+    fields: dict[str, Any] = dict(nested_fields) if isinstance(nested_fields, dict) else {}
+    # Flat extras (everything that isn't "type"/"fields") become fields.
+    for k, v in raw.items():
+        if k in (_HINT_TYPE_KEY, _HINT_FIELD_KEY):
+            continue
+        fields.setdefault(k, v)
+    return PageHint(page_type=_coerce_type(raw_type), fields=fields)

autoai_optimize/analyze/jsdetect.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from bs4 import BeautifulSoup
+
+
+def detect_js_rendered(html: str) -> bool:
+    """Heuristic to detect likely client-side rendered pages.
+
+    Returns True when the HTML appears to be a thin shell and relies on JS to
+    render meaningful content. This is a heuristic (not perfect) and intended
+    to warn users to use prerendering or the offload pre-scan pipeline.
+    """
+    if not html:
+        return False
+    soup = BeautifulSoup(html, "html.parser")
+    body = soup.body
+    # Heuristic rules: very small body text, presence of a root app div, many
+    # <script> tags, or presence of known SPA markers.
+    text_len = len(body.get_text(strip=True)) if body else 0
+    scripts = len(soup.find_all("script"))
+    has_root_div = bool(soup.find(id=lambda x: x and str(x).lower() in ("app", "root", "__next", "svelte-app")))
+    # If body text is tiny and scripts are many, likely JS heavy.
+    if text_len < 50 and scripts >= 2 and has_root_div:
+        return True
+    # If the only text is a loader / noscript fallback, treat as JS-rendered.
+    if body and body.find("noscript") and text_len < 100:
+        return True
+    return False

autoai_optimize/config.py

@@ -0,0 +1,66 @@
+"""Configuration for autoai-optimize.
+
+Zero-config by default: sensible thresholds so the library does the right
+thing out of the box. Override only what you need.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass(frozen=True)
+class Config:
+    """Runtime configuration.
+
+    Attributes:
+        enabled: Master switch. When False, optimize_html returns input untouched.
+        min_confidence: Minimum detection confidence (0.0–1.0) to emit schema.
+            Below this the library stays silent — wrong schema can trigger
+            Google penalties, so silence is always safe.
+        allow_paths: Optional allow-list of URL path prefixes (e.g. ["/blog/"]).
+            When set, only matching paths are processed. Empty = process all.
+        deny_paths: Deny-list of URL path prefixes. Checked before allow_paths.
+        inject_existing: If True and a page already has JSON-LD of the same
+            @type, leave it untouched (idempotent). Recommended True.
+    """
+
+    enabled: bool = True
+    min_confidence: float = 0.5
+    allow_paths: tuple[str, ...] = ()
+    deny_paths: tuple[str, ...] = ()
+    # When True, requires paths listed in sensitive_paths to be explicitly
+    # present in allow_paths before processing. Useful for large sites where
+    # certain prefixes (e.g., /admin/, /user/) must never be auto-scanned
+    # unless explicitly opted-in.
+    require_explicit_opt_in: bool = False
+    sensitive_paths: tuple[str, ...] = ("/admin/", "/dashboard/", "/user/")
+    inject_existing: bool = True
+    api_key: str | None = None
+    webhook_url: str = "https://api.autoai-optimize.com/webhook"
+    ai_endpoint: str = "/api/ai"
+    # Reserved for future per-schema enable flags; kept to avoid config churn.
+    _reserved: tuple[str, ...] = field(default=())
+
+    def path_allowed(self, path: str) -> bool:
+        """Return True if `path` should be processed under this config.
+
+        Order of checks:
+        1. deny_paths (deny overrides everything)
+        2. if require_explicit_opt_in and path matches a sensitive prefix -> only
+           allowed if present in allow_paths
+        3. allow_paths (if empty, allow all non-denied)
+        """
+        if any(path.startswith(p) for p in self.deny_paths):
+            return False
+
+        if self.require_explicit_opt_in and any(path.startswith(p) for p in self.sensitive_paths):
+            # Only allowed when the path is explicitly whitelisted.
+            return any(path.startswith(p) for p in self.allow_paths)
+
+        if not self.allow_paths:
+            return True
+        return any(path.startswith(p) for p in self.allow_paths)
+
+
+DEFAULT_CONFIG = Config()