alterlab 2.1.0__tar.gz → 2.2.0__tar.gz

{alterlab-2.1.0 → alterlab-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alterlab
-Version: 2.1.0
+Version: 2.2.0
 Summary: AlterLab Python SDK
 Author: AlterLab
 Author-email: dev@alterlab.io

{alterlab-2.1.0 → alterlab-2.2.0}/alterlab/__init__.py

@@ -41,7 +41,7 @@ from alterlab.client import (AdvancedOptions, AlterLab, AlterLabAPIError,
 from alterlab.types import (SessionIntegration, SessionListResponse,
                             SessionValidation)
 
-__version__ = "2.1.0"
+__version__ = "2.1.1"
 __all__ = [
     "AlterLab",
     "AlterLabSync",

{alterlab-2.1.0 → alterlab-2.2.0}/alterlab/client.py

@@ -11,93 +11,302 @@ This SDK provides full feature parity with the AlterLab API including:
 - Batch scraping with webhooks
 - Comprehensive error handling with retries
 
-Version: 2.1.0 (Adds session/cookie support for authenticated scraping)
+Version: 2.1.1 (Adds X-AlterLab-Source header for ecosystem attribution)
 """
 
 import asyncio
 import logging
 import time
 import warnings
-from typing import Any, Dict, List, Literal, Optional, Union
+from typing import Any, Dict, List, Literal, Optional, Sequence, Union
 from urllib.parse import urljoin
 
 import httpx
 from alterlab.resources.sessions import AsyncSessions, SyncSessions
 
+_SDK_VERSION = "2.1.1"
+
 logger = logging.getLogger("alterlab")
 
 
+# ---------------------------------------------------------------------------
+# Response type hints (TypedDicts for IDE/type-checker support)
+# ---------------------------------------------------------------------------
+
+
+class TierEscalationDetail(Dict[str, Any]):
+    """Details of a single tier escalation attempt within a scrape request.
+
+    These fields are internal and may not appear in all responses.
+
+    Fields:
+        tier: Tier identifier (e.g. "1", "2", "3", "3.5", "4").
+        result: Outcome — ``"success"``, ``"failed"``, or ``"skipped"``.
+        credits: Credits charged for this tier attempt (microcents).
+        duration_ms: Time taken for this tier attempt in milliseconds.
+        error: Error message if the tier attempt failed.
+    """
+
+
+class BillingDetails(Dict[str, Any]):
+    """Detailed billing breakdown returned in every scrape response.
+
+    Accessed via ``response["billing"]``.
+
+    Fields:
+        total_credits: Total credits charged for the request (microcents).
+        tier_used: Scraping tier that produced the final result (e.g. "1", "4").
+        escalations: List of :class:`TierEscalationDetail` for each tier attempted.
+            Internal — only present when ``x-internal`` fields are not filtered.
+        savings: Credits saved vs. always using the highest tier. Internal.
+        optimization_suggestion: Advisory on how to reduce cost next time. Internal.
+        byop_applied: Whether a Bring-Your-Own-Proxy discount was applied.
+        byop_discount_percent: Discount percentage applied (e.g. ``98.4``).
+        original_cost_microcents: Cost before BYOP discount in microcents.
+        final_cost_microcents: Cost after BYOP discount in microcents.
+        byos_applied: Whether Bring-Your-Own-Session cookies were injected.
+        byos_session_name: Name of the stored session used (e.g. ``"My Amazon Prime"``).
+    """
+
+
+# Valid keys and content types for section_filter validation.
+_SECTION_FILTER_VALID_KEYS = {
+    "min_content_blocks",
+    "exclude_content_types",
+    "content_only",
+}
+_SECTION_FILTER_VALID_CONTENT_TYPES = {
+    "paragraph",
+    "list",
+    "table",
+    "image",
+    "code",
+    "blockquote",
+    "dl",
+    "details",
+}
+
+
 class AdvancedOptions:
     """Advanced scraping options with credit costs.
 
     Credit Costs:
-    - render_js: +3 credits (forces mode="js")
+    - render_js: Forces Tier 4 minimum (browser rendering) — Tier 4 cost applies, no separate add-on charge
     - screenshot: +1 credit (requires render_js)
     - generate_pdf: +2 credits (requires render_js)
     - ocr: +5 credits (refunded if no images found)
     - use_proxy: +1 credit
     - markdown: Free
     - wait_condition: Free
+    - remove_cookie_banners: Free
     """
 
     def __init__(
         self,
-        render_js: bool = False,
+        render_js: Union[bool, Literal["auto"]] = False,
         screenshot: bool = False,
         markdown: bool = False,
         generate_pdf: bool = False,
         ocr: bool = False,
         use_proxy: bool = False,
+        use_own_proxy: bool = False,
+        use_system_proxy: bool = False,
+        proxy_integration_id: Optional[str] = None,
+        proxy_country: Optional[str] = None,
+        remove_cookie_banners: bool = True,
         wait_condition: str = "networkidle",
+        scroll_to_load: bool = False,
+        scroll_count: int = 3,
+        include_iframes: bool = False,
+        flatten_shadow_dom: bool = False,
+        section_filter: Optional[Dict[str, Any]] = None,
+        session_headers: Optional[Dict[str, str]] = None,
     ):
         """Initialize advanced options.
 
         Args:
-            render_js: Render JavaScript using headless browser (+3 credits)
-            screenshot: Capture full-page screenshot (+1 credit, requires render_js)
+            render_js: Render JavaScript using headless browser (forces Tier 4 minimum — no separate
+                add-on charge, Tier 4 cost applies). Set to ``"auto"`` for smart detection: probes
+                each page and only renders JS-heavy pages with a browser, using fast HTTP for static
+                pages. Saves 30–60% on mixed sites. Note: screenshot and generate_pdf require
+                ``render_js=True`` (not ``"auto"``), since they need guaranteed browser rendering.
+            screenshot: Capture full-page screenshot (+1 credit, requires render_js=True)
             markdown: Extract content as Markdown (free)
-            generate_pdf: Generate PDF of rendered page (+2 credits, requires render_js)
+            generate_pdf: Generate PDF of rendered page (+2 credits, requires render_js=True)
             ocr: Extract text from images using OCR (+5 credits)
             use_proxy: Route request through premium proxy (+1 credit)
+            use_own_proxy: Use your own integrated proxy instead of the system proxy
+            use_system_proxy: Override your default proxy integration and use AlterLab's system
+                proxy instead
+            proxy_integration_id: Specific proxy integration ID to use (requires
+                use_own_proxy=True)
+            proxy_country: Preferred proxy country code (e.g., ``"US"``, ``"DE"``) for
+                geo-targeting
+            remove_cookie_banners: Remove cookie consent banners from HTML before content
+                extraction (free, enabled by default)
             wait_condition: Wait condition for JS rendering (domcontentloaded|networkidle|load)
+            scroll_to_load: Scroll page to trigger lazy-loaded content (requires render_js)
+            scroll_count: Number of scroll iterations when scroll_to_load is enabled (1-10)
+            include_iframes: Inline iframe content into the main document (free, requires render_js).
+                Same-origin iframes are read directly; cross-origin iframes are marked but not fetched.
+            flatten_shadow_dom: Flatten Shadow DOM roots into regular DOM for extraction (free,
+                requires render_js). Web Components with shadow roots become visible in the
+                serialized HTML.
+            section_filter: Filter options for json_v2 section tree output. Dict with optional keys:
+                - min_content_blocks (int): Minimum content blocks for a section (default 0)
+                - exclude_content_types (list[str]): Content types to exclude
+                  (paragraph, list, table, image, code, blockquote, dl, details)
+                - content_only (bool): Strip headings, keep only content (default False)
+            session_headers: Inline auth headers (e.g. {"Authorization": "Bearer token"}).
+                Merged with session headers when session_id is also provided.
 
         Raises:
-            ValueError: If screenshot or generate_pdf is True but render_js is False
+            ValueError: If render_js is not a bool or the string ``"auto"``
+            ValueError: If screenshot or generate_pdf is True but render_js is not True
+            ValueError: If scroll_to_load is True but render_js is falsy
+            ValueError: If include_iframes or flatten_shadow_dom is True but render_js is falsy
+            ValueError: If scroll_count is not between 1 and 10
+            ValueError: If section_filter contains unknown keys or invalid value types
         """
-        if screenshot and not render_js:
-            raise ValueError("screenshot requires render_js=True")
-        if generate_pdf and not render_js:
-            raise ValueError("generate_pdf requires render_js=True")
+        if render_js not in (True, False, "auto"):
+            raise ValueError('render_js must be True, False, or "auto"')
+        if screenshot and render_js is not True:
+            raise ValueError(
+                "screenshot requires render_js=True "
+                '(render_js="auto" is not sufficient — screenshot needs guaranteed browser rendering)'
+            )
+        if generate_pdf and render_js is not True:
+            raise ValueError(
+                "generate_pdf requires render_js=True "
+                '(render_js="auto" is not sufficient — PDF generation needs guaranteed browser rendering)'
+            )
+        if scroll_to_load and not render_js:
+            raise ValueError(
+                "scroll_to_load requires render_js=True or render_js='auto'"
+            )
+        if include_iframes and not render_js:
+            raise ValueError(
+                "include_iframes requires render_js=True or render_js='auto'"
+            )
+        if flatten_shadow_dom and not render_js:
+            raise ValueError(
+                "flatten_shadow_dom requires render_js=True or render_js='auto'"
+            )
 
         valid_conditions = ["domcontentloaded", "networkidle", "load"]
         if wait_condition not in valid_conditions:
             raise ValueError(f"wait_condition must be one of {valid_conditions}")
 
+        if not 1 <= scroll_count <= 10:
+            raise ValueError("scroll_count must be between 1 and 10")
+
         self.render_js = render_js
         self.screenshot = screenshot
         self.markdown = markdown
         self.generate_pdf = generate_pdf
         self.ocr = ocr
         self.use_proxy = use_proxy
+        self.use_own_proxy = use_own_proxy
+        self.use_system_proxy = use_system_proxy
+        self.proxy_integration_id = proxy_integration_id
+        self.proxy_country = proxy_country
+        self.remove_cookie_banners = remove_cookie_banners
         self.wait_condition = wait_condition
+        self.scroll_to_load = scroll_to_load
+        self.scroll_count = scroll_count
+        self.include_iframes = include_iframes
+        self.flatten_shadow_dom = flatten_shadow_dom
+        self.session_headers = session_headers
+
+        if section_filter is not None:
+            self._validate_section_filter(section_filter)
+        self.section_filter = section_filter
+
+    @staticmethod
+    def _validate_section_filter(sf: Dict[str, Any]) -> None:
+        """Validate section_filter keys and value types.
+
+        Raises:
+            ValueError: If any key is not in the allowed set, or if value
+                types do not match the expected schema.
+        """
+        if not isinstance(sf, dict):
+            raise ValueError("section_filter must be a dict")
+
+        unknown_keys = set(sf.keys()) - _SECTION_FILTER_VALID_KEYS
+        if unknown_keys:
+            raise ValueError(
+                f"section_filter contains unknown keys: {sorted(unknown_keys)}. "
+                f"Allowed keys: {sorted(_SECTION_FILTER_VALID_KEYS)}"
+            )
+
+        if "min_content_blocks" in sf:
+            val = sf["min_content_blocks"]
+            if not isinstance(val, int) or isinstance(val, bool) or val < 0:
+                raise ValueError(
+                    "section_filter['min_content_blocks'] must be a non-negative integer"
+                )
+
+        if "exclude_content_types" in sf:
+            val = sf["exclude_content_types"]
+            if not isinstance(val, (list, tuple)):
+                raise ValueError(
+                    "section_filter['exclude_content_types'] must be a list of strings"
+                )
+            invalid = set(val) - _SECTION_FILTER_VALID_CONTENT_TYPES
+            if invalid:
+                raise ValueError(
+                    f"section_filter['exclude_content_types'] contains unknown types: "
+                    f"{sorted(invalid)}. "
+                    f"Allowed: {sorted(_SECTION_FILTER_VALID_CONTENT_TYPES)}"
+                )
+
+        if "content_only" in sf:
+            val = sf["content_only"]
+            if not isinstance(val, bool):
+                raise ValueError("section_filter['content_only'] must be a bool")
 
     def to_dict(self) -> Dict[str, Any]:
         """Convert to dictionary for API request."""
-        return {
+        result: Dict[str, Any] = {
             "render_js": self.render_js,
             "screenshot": self.screenshot,
             "markdown": self.markdown,
             "generate_pdf": self.generate_pdf,
             "ocr": self.ocr,
             "use_proxy": self.use_proxy,
+            "use_own_proxy": self.use_own_proxy,
+            "use_system_proxy": self.use_system_proxy,
+            "remove_cookie_banners": self.remove_cookie_banners,
             "wait_condition": self.wait_condition,
+            "scroll_to_load": self.scroll_to_load,
+            "scroll_count": self.scroll_count,
+            "include_iframes": self.include_iframes,
+            "flatten_shadow_dom": self.flatten_shadow_dom,
         }
+        if self.scroll_to_load:
+            result["scroll_to_load"] = True
+            result["scroll_count"] = self.scroll_count
+        if self.section_filter is not None:
+            result["section_filter"] = self.section_filter
+        if self.session_headers is not None:
+            result["session_headers"] = self.session_headers
+        if self.proxy_integration_id is not None:
+            result["proxy_integration_id"] = self.proxy_integration_id
+        if self.proxy_country is not None:
+            result["proxy_country"] = self.proxy_country
+        return result
 
     def calculate_credits(self) -> int:
-        """Calculate additional credits for advanced options."""
+        """Calculate additional credits for advanced options.
+
+        Note: render_js="auto" does not add upfront credits — actual cost depends on
+        per-page decision (browser vs. HTTP). Treat as 0 for estimation purposes.
+        """
         credits = 0
-        if self.render_js:
+        if self.render_js is True:
             credits += 3
+        # render_js="auto" doesn't add credits upfront — actual cost depends on per-page decision
         if self.screenshot:
             credits += 1
         if self.generate_pdf:
@@ -127,7 +336,7 @@ class CostControls:
 
         Args:
             max_credits: Maximum credits to spend on this request
-            max_tier: Maximum tier to escalate to (0.5, 1, 1.5, 2, 3, 4)
+            max_tier: Maximum tier to escalate to (1, 2, 3, 3.5, 4)
             prefer_cost: Optimize for cost (try cheaper tiers first)
             prefer_speed: Optimize for speed (skip to reliable tier)
             fail_fast: Return error instead of escalating to expensive tiers
@@ -135,7 +344,7 @@ class CostControls:
         Raises:
             ValueError: If max_tier is not a valid tier value
         """
-        valid_tiers = ["0.5", "1", "1.5", "2", "3", "4"]
+        valid_tiers = ["1", "2", "3", "3.5", "4"]
         if max_tier and max_tier not in valid_tiers:
             raise ValueError(f"max_tier must be one of {valid_tiers}")
 
@@ -262,7 +471,8 @@ class AlterLab:
             base_url=self.base_url,
             headers={
                 "X-API-Key": api_key,
-                "User-Agent": "AlterLab-Python-SDK/2.1.0",
+                "User-Agent": f"AlterLab-Python-SDK/{_SDK_VERSION}",
+                "X-AlterLab-Source": f"python-sdk/{_SDK_VERSION}",
                 "Content-Type": "application/json",
             },
             timeout=httpx.Timeout(timeout),
@@ -417,6 +627,13 @@ class AlterLab:
         ocr_language: str = "eng",
         session_id: Optional[str] = None,
         cookies: Optional[Dict[str, str]] = None,
+        actions: Optional[List[Dict[str, Any]]] = None,
+        headers: Optional[Dict[str, str]] = None,
+        location: Optional[Dict[str, str]] = None,
+        cache: bool = False,
+        cache_ttl: Optional[int] = None,
+        webhook_url: Optional[str] = None,
+        max_response_bytes: Optional[int] = None,
         poll_interval: float = 1.0,
         poll_timeout: float = 300.0,
         poll_backoff_multiplier: float = 1.5,
@@ -452,18 +669,76 @@ class AlterLab:
                 Mutually exclusive with ``cookies``.
             cookies: Inline cookie dict for one-off authenticated scraping (not stored).
                 Mutually exclusive with ``session_id``.
+            actions: Browser automation sequence to execute after page load (max 20 actions).
+                Requires ``advanced.render_js=True``. Each action is a dict with a ``type``
+                discriminator. Supported types: ``wait``, ``wait_for``, ``click``, ``fill``,
+                ``press``, ``scroll``, ``infinite_scroll``, ``screenshot``, ``evaluate``.
+                Actions execute sequentially; failed actions are skipped. Results returned in
+                ``action_results``. Examples::
+
+                    [
+                        {"type": "wait", "milliseconds": 1000},
+                        {"type": "click", "selector": "button.load-more"},
+                        {"type": "scroll", "direction": "down", "amount": 3},
+                        {"type": "evaluate", "script": "document.title"},
+                    ]
+
+            headers: Custom HTTP request headers to forward to the target URL (max 50 headers).
+                Header names must be 1-256 characters and header values must be at most 8192
+                characters. Hop-by-hop and connection-control headers (e.g. ``Connection``,
+                ``Transfer-Encoding``) are rejected by the API. Example::
+
+                    {"Authorization": "Bearer token", "X-Custom-Header": "value"}
+
+            location: Geo-targeting parameters. Dict with optional keys:
+                - ``country`` (str): ISO 3166-1 alpha-2 country code (e.g., ``"DE"``, ``"JP"``).
+                  Routes the request through a proxy in that country.
+                - ``language`` (str): ISO 639-1 language code (e.g., ``"de"``, ``"ja"``).
+                  Sets the ``Accept-Language`` header and browser locale.
+                Example: ``{"country": "DE", "language": "de"}``
+            cache: Enable response caching. When ``True``, a cached result may be returned if
+                available. Defaults to ``False`` (no caching). Use ``force_refresh=True`` to bypass
+                an existing cache entry.
+            cache_ttl: Cache TTL in seconds (60–86400). Only applies when ``cache=True``.
+                Defaults to 3600 (1 hour) server-side when not specified.
+            webhook_url: URL to receive a ``job.completed`` or ``job.failed`` event when the
+                scrape finishes. This is a server-side notification side-effect and does not
+                affect client-side polling behavior — use ``sync=False`` to skip polling.
+                HTTPS is required in production (``http://localhost`` is allowed in dev).
+            max_response_bytes: Soft cap on raw response body size in bytes. When the
+                downloaded HTML exceeds this value it is truncated before extraction and
+                ``content_truncated.truncation_reason`` is set to ``'response_body_cap'``.
+                Default: ``None`` (server default of 5 MB applies). Set to ``0`` for no
+                limit. Maximum: 50 MB (52_428_800).
             poll_interval: Initial polling interval for async jobs (seconds)
             poll_timeout: Maximum time to wait for async jobs (seconds)
             poll_backoff_multiplier: Multiplier for exponential backoff (default: 1.5)
             poll_max_interval: Maximum polling interval (seconds)
 
         Returns:
-            Scrape response dictionary with content, metadata, and billing details
+            Scrape response dictionary with content, metadata, and billing details.
+            Key fields include:
+            - ``url``, ``status_code``, ``content``, ``headers``, ``cached``
+            - ``credits_used``, ``response_time_ms``, ``size_bytes``
+            - ``billing`` (:class:`BillingDetails`): Full billing breakdown including
+              ``total_credits``, ``tier_used``, ``escalations``
+              (``List[TierEscalationDetail]`` — each entry has ``tier``, ``result``,
+              ``credits: int``, ``duration_ms: Optional[int]``, ``error:
+              Optional[str]``), ``byop_applied``, and BYOP/BYOS discount fields.
+            - ``warning`` (Optional[str]): Actionable warning when auto-escalation
+              occurred, tier cap prevented escalation on thin content, or extraction
+              quality is low.
+            - ``content_quality`` (Optional[Dict]): Content quality metadata from
+              scrape diagnostics (e.g., word count, image count, quality score).
+            - ``note`` (Optional[str]): Advisory message (e.g. when repeat-request
+              detection identifies identical content across retries).
 
         Raises:
             AlterLabAPIError: On API errors
             AlterLabTimeoutError: If async job polling times out
             AlterLabValidationError: If parameters are invalid
+            AlterLabValidationError: If actions is provided but advanced.render_js is not set
+            AlterLabValidationError: If more than 20 actions are provided
 
         Example:
             # Simple HTML scraping
@@ -499,6 +774,33 @@ class AlterLab:
                 "https://amazon.com/dp/B0XXXXX",
                 session_id="uuid-of-stored-session"
             )
+
+            # Geo-targeted scraping — retrieve German content via DE proxy
+            result = await client.scrape(
+                "https://example.com",
+                location={"country": "DE", "language": "de"}
+            )
+
+            # Smart JS detection — renders only JS-heavy pages (30-60% cost savings)
+            result = await client.scrape(
+                "https://example.com",
+                advanced=AdvancedOptions(render_js="auto")
+            )
+
+            # Cache-enabled scrape with custom TTL
+            result = await client.scrape(
+                "https://example.com",
+                cache=True,
+                cache_ttl=7200  # 2-hour cache
+            )
+
+            # Async scrape with webhook delivery (skips polling)
+            result = await client.scrape(
+                "https://example.com",
+                sync=False,
+                webhook_url="https://myapp.com/hooks/scrape-done"
+            )
+            # result contains {"job_id": "..."} — webhook delivers final result
         """
         if session_id and cookies:
             raise AlterLabValidationError(
@@ -506,6 +808,27 @@ class AlterLab:
                 "Use session_id to reference a stored session OR "
                 "provide inline cookies, not both."
             )
+        if actions is not None:
+            if not advanced or not advanced.render_js:
+                raise AlterLabValidationError(
+                    "actions requires advanced.render_js=True or render_js='auto' "
+                    "(browser rendering must be enabled to execute browser actions)"
+                )
+            if len(actions) > 20:
+                raise AlterLabValidationError(
+                    f"actions must contain at most 20 items, got {len(actions)}"
+                )
+        if screenshot and (advanced is None or advanced.render_js is not True):
+            raise AlterLabValidationError(
+                "screenshot requires render_js=True. "
+                "Pass advanced=AdvancedOptions(render_js=True, screenshot=True) "
+                "or use scrape_js(screenshot=True) instead."
+            )
+        if cache_ttl is not None and not cache:
+            raise AlterLabValidationError(
+                "cache_ttl requires cache=True. "
+                "Pass cache=True to enable caching, or omit cache_ttl."
+            )
 
         # Build request payload
         payload: Dict[str, Any] = {
@@ -530,6 +853,8 @@ class AlterLab:
             payload["advanced"]["session_id"] = session_id
         if cookies:
             payload["advanced"]["cookies"] = cookies
+        if actions:
+            payload["advanced"]["actions"] = actions
 
         # Remove empty advanced block
         if not payload["advanced"]:
@@ -565,6 +890,28 @@ class AlterLab:
         if mode == "ocr":
             payload["ocr_language"] = ocr_language
 
+        # Custom request headers forwarded to target URL
+        if headers is not None:
+            payload["headers"] = headers
+
+        # Geo-targeting
+        if location is not None:
+            payload["location"] = location
+
+        # Cache control
+        if cache:
+            payload["cache"] = cache
+            if cache_ttl is not None:
+                payload["cache_ttl"] = cache_ttl
+
+        # Webhook URL for async notification
+        if webhook_url is not None:
+            payload["webhook_url"] = webhook_url
+
+        # Response body size cap
+        if max_response_bytes is not None:
+            payload["max_response_bytes"] = max_response_bytes
+
         # Make request
         response = await self._request_with_retry(
             "POST", "/api/v1/scrape", json=payload
@@ -575,7 +922,10 @@ class AlterLab:
         # Check if response is async (202 Accepted with job_id)
         if response.status_code == 202 and "job_id" in result:
             if not sync:
-                # Return job info immediately
+                # Return job info immediately in async mode.
+                # When webhook_url is also provided alongside sync=True, the server
+                # delivers the result via webhook as a side-effect, but polling still
+                # proceeds so the caller receives the scraped content directly.
                 return result
 
             # Poll for job completion with exponential backoff
@@ -1080,6 +1430,91 @@ class AlterLab:
 
         return response.json()
 
+    async def crawl_stream(
+        self,
+        crawl_id: str,
+    ):
+        """Stream real-time crawl events via Server-Sent Events.
+
+        Returns an async iterator that yields parsed SSE events as
+        dictionaries with ``event`` (str) and ``data`` (dict) keys.
+
+        Event types:
+        - ``page.completed`` — per-page result with metadata, cost, tier
+        - ``page.failed`` — per-page failure with error reason
+        - ``crawl.progress`` — periodic summary (pages done/total, cost)
+        - ``crawl.completed`` — final summary; iteration stops after this
+
+        Args:
+            crawl_id: Crawl ID returned from crawl()
+
+        Yields:
+            Dict with 'event' (str) and 'data' (dict) keys
+
+        Raises:
+            AlterLabAPIError: On HTTP errors (404 if crawl not found)
+
+        Example:
+            crawl = await client.crawl("https://example.com", max_pages=10)
+            async for event in client.crawl_stream(crawl["crawl_id"]):
+                if event["event"] == "page.completed":
+                    print(f"Scraped: {event['data']['url']}")
+                elif event["event"] == "crawl.completed":
+                    print(f"Done: {event['data']['completed']}/{event['data']['total']}")
+        """
+        import json as _json
+
+        url = f"/api/v1/crawl/{crawl_id}/stream"
+        headers = {
+            "X-API-Key": self._api_key,
+            "Accept": "text/event-stream",
+        }
+
+        async with httpx.AsyncClient(
+            base_url=self._base_url, timeout=None
+        ) as stream_client:
+            async with stream_client.stream("GET", url, headers=headers) as response:
+                if response.status_code != 200:
+                    body = await response.aread()
+                    raise AlterLabAPIError(
+                        f"SSE stream failed: {response.status_code}",
+                        status_code=response.status_code,
+                        response_body=body.decode("utf-8", errors="replace"),
+                    )
+
+                current_event = "message"
+                data_lines: list[str] = []
+
+                async for raw_line in response.aiter_lines():
+                    line = raw_line.rstrip("\n").rstrip("\r")
+
+                    # Comment or keepalive
+                    if line.startswith(":"):
+                        continue
+
+                    # Empty line = event boundary
+                    if not line:
+                        if data_lines:
+                            data_str = "\n".join(data_lines)
+                            try:
+                                parsed = _json.loads(data_str)
+                            except _json.JSONDecodeError:
+                                parsed = data_str
+                            yield {"event": current_event, "data": parsed}
+
+                            # Terminal event — stop iteration
+                            if current_event == "crawl.completed":
+                                return
+
+                            current_event = "message"
+                            data_lines = []
+                        continue
+
+                    if line.startswith("event:"):
+                        current_event = line[6:].strip()
+                    elif line.startswith("data:"):
+                        data_lines.append(line[5:].strip())
+
     async def crawl_and_wait(
         self,
         url: str,
@@ -1186,6 +1621,280 @@ class AlterLab:
                 current_interval * poll_backoff_multiplier, poll_max_interval
             )
 
+    # ------------------------------------------------------------------
+    # Search
+    # ------------------------------------------------------------------
+
+    async def search(
+        self,
+        query: str,
+        *,
+        domain: Optional[str] = None,
+        num_results: int = 10,
+        page: int = 1,
+        country: Optional[str] = None,
+        language: Optional[str] = None,
+        time_range: Optional[Literal["hour", "day", "week", "month", "year"]] = None,
+        scrape_results: bool = False,
+        formats: Optional[
+            List[Literal["text", "json", "json_v2", "html", "markdown"]]
+        ] = None,
+        extraction_schema: Optional[Dict[str, Any]] = None,
+    ) -> Dict[str, Any]:
+        """Execute a web search and return SERP results.
+
+        Args:
+            query: Search terms (max 500 characters)
+            domain: Restrict results to a specific domain (applied as site: prefix)
+            num_results: Number of results to return (1-30, default 10)
+            page: Result page number (1-indexed, default 1)
+            country: ISO 3166-1 alpha-2 country code for geo-targeted results
+            language: Language code for results (e.g., en, fr, de)
+            time_range: Filter results by recency
+            scrape_results: If True, scrape each result page and include content
+            formats: Output formats when scrape_results=True
+            extraction_schema: JSON schema for structured extraction (when scrape_results=True)
+
+        Returns:
+            Search response with search_id, results, credits_used, etc.
+
+        Raises:
+            AlterLabAPIError: On API errors
+
+        Example:
+            results = await client.search("web scraping api python")
+            for r in results["results"]:
+                print(f"{r['position']}. {r['title']} — {r['url']}")
+        """
+        payload: Dict[str, Any] = {
+            "query": query,
+            "num_results": num_results,
+            "page": page,
+            "scrape_results": scrape_results,
+        }
+        if domain is not None:
+            payload["domain"] = domain
+        if country is not None:
+            payload["country"] = country
+        if language is not None:
+            payload["language"] = language
+        if time_range is not None:
+            payload["time_range"] = time_range
+        if formats is not None:
+            payload["formats"] = formats
+        if extraction_schema is not None:
+            payload["extraction_schema"] = extraction_schema
+
+        response = await self._request_with_retry(
+            "POST", "/api/v1/search", json=payload
+        )
+        return response.json()
+
+    async def batch_search(
+        self,
+        queries: List[str],
+        *,
+        num_results: int = 10,
+        country: Optional[str] = None,
+        language: Optional[str] = None,
+        time_range: Optional[Literal["hour", "day", "week", "month", "year"]] = None,
+        scrape_results: bool = False,
+        formats: Optional[
+            List[Literal["text", "json", "json_v2", "html", "markdown"]]
+        ] = None,
+        webhook_url: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """Submit a batch of search queries for async parallel processing.
+
+        Args:
+            queries: List of search queries (2-100)
+            num_results: Number of results per query (1-30, default 10)
+            country: ISO 3166-1 alpha-2 country code
+            language: Language code
+            time_range: Filter results by recency
+            scrape_results: Scrape each result page
+            formats: Output formats when scrape_results=True
+            webhook_url: URL to receive batch.search.completed webhook
+
+        Returns:
+            Batch response with batch_id, job_ids, status, etc.
+
+        Raises:
+            AlterLabAPIError: On API errors
+        """
+        payload: Dict[str, Any] = {
+            "queries": queries,
+            "num_results": num_results,
+            "scrape_results": scrape_results,
+        }
+        if country is not None:
+            payload["country"] = country
+        if language is not None:
+            payload["language"] = language
+        if time_range is not None:
+            payload["time_range"] = time_range
+        if formats is not None:
+            payload["formats"] = formats
+        if webhook_url is not None:
+            payload["webhook_url"] = webhook_url
+
+        response = await self._request_with_retry(
+            "POST", "/api/v1/search/batch", json=payload
+        )
+        return response.json()
+
+    # ------------------------------------------------------------------
+    # Map
+    # ------------------------------------------------------------------
+
+    async def map(
+        self,
+        url: str,
+        *,
+        max_pages: int = 500,
+        max_depth: int = 3,
+        include_patterns: Optional[List[str]] = None,
+        exclude_patterns: Optional[List[str]] = None,
+        search: Optional[str] = None,
+        sitemap: Literal["skip", "include", "only"] = "include",
+        sitemap_path: Optional[str] = None,
+        include_metadata: bool = False,
+        include_subdomains: bool = False,
+        respect_robots: bool = True,
+    ) -> Dict[str, Any]:
+        """Discover URLs on a website via sitemap parsing and link extraction.
+
+        Lightweight site structure discovery without scraping content.
+        Returns a flat list of discovered URLs with optional metadata.
+
+        Args:
+            url: Starting URL for site discovery
+            max_pages: Maximum URLs to discover (1-50000, default 500)
+            max_depth: Link-following depth (0-10, default 3)
+            include_patterns: Glob patterns — only include matching URLs
+            exclude_patterns: Glob patterns — exclude matching URLs
+            search: Query to filter/rank discovered URLs by relevance
+            sitemap: Sitemap handling: skip, include (default), or only
+            sitemap_path: Custom sitemap location (e.g., /custom-sitemap.xml)
+            include_metadata: Fetch title and meta description for each URL
+            include_subdomains: Include URLs from subdomains
+            respect_robots: Respect robots.txt directives (default True)
+
+        Returns:
+            Map response with map_id, urls, total_urls, credits_used, etc.
+
+        Raises:
+            AlterLabAPIError: On API errors
+
+        Example:
+            result = await client.map(
+                "https://example.com",
+                max_pages=1000,
+                include_metadata=True,
+            )
+            for entry in result["urls"]:
+                print(f"{entry['url']} (depth={entry['depth']})")
+        """
+        payload: Dict[str, Any] = {
+            "url": url,
+            "max_pages": max_pages,
+            "max_depth": max_depth,
+            "sitemap": sitemap,
+            "include_metadata": include_metadata,
+            "include_subdomains": include_subdomains,
+            "respect_robots": respect_robots,
+        }
+        if include_patterns is not None:
+            payload["include_patterns"] = include_patterns
+        if exclude_patterns is not None:
+            payload["exclude_patterns"] = exclude_patterns
+        if search is not None:
+            payload["search"] = search
+        if sitemap_path is not None:
+            payload["sitemap_path"] = sitemap_path
+
+        response = await self._request_with_retry("POST", "/api/v1/map", json=payload)
+        return response.json()
+
+    # ------------------------------------------------------------------
+    # Extract
+    # ------------------------------------------------------------------
+
+    async def extract(
+        self,
+        content: str,
+        *,
+        content_type: Literal["html", "text", "markdown"] = "html",
+        extraction_schema: Optional[Dict[str, Any]] = None,
+        extraction_profile: Optional[
+            Literal[
+                "auto",
+                "product",
+                "article",
+                "job_posting",
+                "faq",
+                "recipe",
+                "event",
+            ]
+        ] = None,
+        extraction_prompt: Optional[str] = None,
+        formats: Optional[
+            List[Literal["text", "json", "json_v2", "html", "markdown", "rag"]]
+        ] = None,
+        source_url: Optional[str] = None,
+        evidence: bool = False,
+    ) -> Dict[str, Any]:
+        """Extract structured data from raw content without scraping.
+
+        Accepts HTML, text, or markdown content and runs the extraction pipeline
+        including format conversion, schema-based filtering, and LLM extraction.
+
+        Args:
+            content: Raw content to extract from (HTML, text, or markdown)
+            content_type: Type of the provided content (default "html")
+            extraction_schema: JSON Schema for structured extraction
+            extraction_profile: Pre-defined extraction profile
+            extraction_prompt: Natural language extraction instructions for LLM
+            formats: Output formats (default ["json"])
+            source_url: Original URL of the content (for context, not fetched)
+            evidence: Include field provenance/evidence (default False)
+
+        Returns:
+            Extract response with extract_id, formats, credits_used, etc.
+
+        Raises:
+            AlterLabAPIError: On API errors
+
+        Example:
+            result = await client.extract(
+                content="<html><body><h1>Price: $29.99</h1></body></html>",
+                content_type="html",
+                extraction_prompt="Extract the price",
+                formats=["json"],
+            )
+            print(result["formats"]["json"])
+        """
+        payload: Dict[str, Any] = {
+            "content": content,
+            "content_type": content_type,
+            "evidence": evidence,
+        }
+        if extraction_schema is not None:
+            payload["extraction_schema"] = extraction_schema
+        if extraction_profile is not None:
+            payload["extraction_profile"] = extraction_profile
+        if extraction_prompt is not None:
+            payload["extraction_prompt"] = extraction_prompt
+        if formats is not None:
+            payload["formats"] = formats
+        if source_url is not None:
+            payload["source_url"] = source_url
+
+        response = await self._request_with_retry(
+            "POST", "/api/v1/extract", json=payload
+        )
+        return response.json()
+
     # Deprecated methods for backwards compatibility
 
     async def scrape_light(self, url: str, **kwargs) -> Dict[str, Any]:
@@ -1307,6 +2016,22 @@ class AlterLabSync:
         """Synchronous version of crawl_and_wait()."""
         return self._run(self._client.crawl_and_wait(*args, **kwargs))
 
+    def search(self, *args, **kwargs) -> Dict[str, Any]:
+        """Synchronous version of search()."""
+        return self._run(self._client.search(*args, **kwargs))
+
+    def batch_search(self, *args, **kwargs) -> Dict[str, Any]:
+        """Synchronous version of batch_search()."""
+        return self._run(self._client.batch_search(*args, **kwargs))
+
+    def map(self, *args, **kwargs) -> Dict[str, Any]:
+        """Synchronous version of map()."""
+        return self._run(self._client.map(*args, **kwargs))
+
+    def extract(self, *args, **kwargs) -> Dict[str, Any]:
+        """Synchronous version of extract()."""
+        return self._run(self._client.extract(*args, **kwargs))
+
     def scrape_html(self, *args, **kwargs) -> Dict[str, Any]:
         """Synchronous version of scrape_html()."""
         return self._run(self._client.scrape_html(*args, **kwargs))

alterlab-2.2.0/alterlab/types.py

@@ -0,0 +1,93 @@
+"""Type definitions for AlterLab SDK session and cookie support."""
+
+from typing import Any, Dict, List, Literal, Optional
+
+from typing_extensions import TypedDict
+
+
+class TierInfo(TypedDict):
+    """Rate-limit tier information derived from account balance.
+
+    Returned as part of the ``/auth/me`` response so that SDK consumers can
+    inspect their current tier without a separate API call.
+
+    Balance-based tier table:
+
+    +--------------+-----------+-------+-------------+
+    | Tier         | Balance   |  RPM  | Concurrent  |
+    +==============+===========+=======+=============+
+    | Free         | $0–$5     |  30   |      3      |
+    | Basic        | $5–$50    |  60   |     10      |
+    | Growth       | $50–$200  | 100   |     15      |
+    | Scale        | $200–$500 | 300   |     25      |
+    | High Volume  | $500+     | 500   |     50      |
+    | Enterprise   | Custom    | 10000 |    500      |
+    +--------------+-----------+-------+-------------+
+    """
+
+    name: str
+    """Human-readable tier name, e.g. ``'Free'``, ``'Basic'``, ``'Growth'``."""
+
+    slug: str
+    """Machine-readable tier identifier, e.g. ``'free'``, ``'basic'``, ``'growth'``."""
+
+    requests_per_minute: int
+    """Maximum API requests per minute allowed at this tier."""
+
+    concurrent_requests: int
+    """Maximum number of concurrent scrape requests allowed at this tier."""
+
+    min_balance_usd: float
+    """Minimum account balance (USD) required to qualify for this tier."""
+
+    max_balance_usd: Optional[float]
+    """Upper balance threshold (USD) for this tier. ``None`` means no upper limit."""
+
+    next_tier_slug: Optional[str]
+    """Slug of the next higher tier, or ``None`` if already at the top tier."""
+
+    next_tier_threshold_usd: Optional[float]
+    """Balance (USD) needed to reach the next tier, or ``None`` if at top tier."""
+
+
+class SessionIntegration(TypedDict, total=False):
+    """A stored session integration. Cookie values are never exposed."""
+
+    id: str
+    name: str
+    domain: str
+    cookie_names: List[str]
+    header_names: Optional[List[str]]
+    status: Literal["active", "expired", "invalid", "pending_validation"]
+    expires_at: Optional[str]
+    last_validated_at: Optional[str]
+    validation_confidence: Optional[int]
+    consecutive_failures: int
+    last_used_at: Optional[str]
+    total_requests: int
+    successful_requests: int
+    success_rate: float
+    notes: Optional[str]
+    created_at: str
+    updated_at: str
+
+
+class SessionListResponse(TypedDict):
+    """Response from listing sessions."""
+
+    sessions: List[SessionIntegration]
+    total: int
+
+
+class SessionValidation(TypedDict, total=False):
+    """Result of a session validation check."""
+
+    is_valid: bool
+    confidence: int
+    http_status: Optional[int]
+    detected_user: Optional[str]
+    cookie_expiry_detected: Optional[str]
+    cookies_expiring_soon: bool
+    indicators_found: Optional[Dict[str, List[str]]]
+    consecutive_failures: int
+    details: str

{alterlab-2.1.0 → alterlab-2.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "alterlab"
-version = "2.1.0"
+version = "2.2.0"
 description = "AlterLab Python SDK"
 authors = ["AlterLab <dev@alterlab.io>"]
 

alterlab-2.1.0/alterlab/types.py

@@ -1,48 +0,0 @@
-"""Type definitions for AlterLab SDK session and cookie support."""
-
-from typing import Any, Dict, List, Literal, Optional
-
-from typing_extensions import TypedDict
-
-
-class SessionIntegration(TypedDict, total=False):
-    """A stored session integration. Cookie values are never exposed."""
-
-    id: str
-    name: str
-    domain: str
-    cookie_names: List[str]
-    header_names: Optional[List[str]]
-    status: Literal["active", "expired", "invalid", "pending_validation"]
-    expires_at: Optional[str]
-    last_validated_at: Optional[str]
-    validation_confidence: Optional[int]
-    consecutive_failures: int
-    last_used_at: Optional[str]
-    total_requests: int
-    successful_requests: int
-    success_rate: float
-    notes: Optional[str]
-    created_at: str
-    updated_at: str
-
-
-class SessionListResponse(TypedDict):
-    """Response from listing sessions."""
-
-    sessions: List[SessionIntegration]
-    total: int
-
-
-class SessionValidation(TypedDict, total=False):
-    """Result of a session validation check."""
-
-    is_valid: bool
-    confidence: int
-    http_status: Optional[int]
-    detected_user: Optional[str]
-    cookie_expiry_detected: Optional[str]
-    cookies_expiring_soon: bool
-    indicators_found: Optional[Dict[str, List[str]]]
-    consecutive_failures: int
-    details: str