sentienceapi 0.90.17__py3-none-any.whl

sentience/query.py

@@ -0,0 +1,303 @@
+"""
+Query engine v1 - semantic selector matching
+"""
+
+import re
+from typing import Any
+
+from .models import Element, Snapshot
+
+
+def parse_selector(selector: str) -> dict[str, Any]:  # noqa: C901
+    """
+    Parse string DSL selector into structured query
+
+    Examples:
+        "role=button text~'Sign in'"
+        "role=textbox name~'email'"
+        "clickable=true role=link"
+        "role!=link"
+        "importance>500"
+        "text^='Sign'"
+        "text$='in'"
+    """
+    query: dict[str, Any] = {}
+
+    # Match patterns like: key=value, key~'value', key!="value", key>123, key^='prefix', key$='suffix'
+    # Updated regex to support: =, !=, ~, ^=, $=, >, >=, <, <=
+    # Supports dot notation: attr.id, css.color
+    # Note: Handle ^= and $= first (before single char operators) to avoid regex conflicts
+    # Pattern matches: key, operator (including ^= and $=), and value (quoted or unquoted)
+    pattern = r"([\w.]+)(\^=|\$=|>=|<=|!=|[=~<>])((?:\'[^\']+\'|\"[^\"]+\"|[^\s]+))"
+    matches = re.findall(pattern, selector)
+
+    for key, op, value in matches:
+        # Remove quotes from value
+        value = value.strip().strip("\"'")
+
+        # Handle numeric comparisons
+        is_numeric = False
+        try:
+            numeric_value = float(value)
+            is_numeric = True
+        except ValueError:
+            pass
+
+        if op == "!=":
+            if key == "role":
+                query["role_exclude"] = value
+            elif key == "clickable":
+                query["clickable"] = False
+            elif key == "visible":
+                query["visible"] = False
+        elif op == "~":
+            # Substring match (case-insensitive)
+            if key == "text" or key == "name":
+                query["text_contains"] = value
+        elif op == "^=":
+            # Prefix match
+            if key == "text" or key == "name":
+                query["text_prefix"] = value
+        elif op == "$=":
+            # Suffix match
+            if key == "text" or key == "name":
+                query["text_suffix"] = value
+        elif op == ">":
+            # Greater than
+            if is_numeric:
+                if key == "importance":
+                    query["importance_min"] = numeric_value + 0.0001  # Exclusive
+                elif key.startswith("bbox."):
+                    query[f"{key}_min"] = numeric_value + 0.0001
+                elif key == "z_index":
+                    query["z_index_min"] = numeric_value + 0.0001
+            elif key.startswith("attr.") or key.startswith("css."):
+                query[f"{key}_gt"] = value
+        elif op == ">=":
+            # Greater than or equal
+            if is_numeric:
+                if key == "importance":
+                    query["importance_min"] = numeric_value
+                elif key.startswith("bbox."):
+                    query[f"{key}_min"] = numeric_value
+                elif key == "z_index":
+                    query["z_index_min"] = numeric_value
+            elif key.startswith("attr.") or key.startswith("css."):
+                query[f"{key}_gte"] = value
+        elif op == "<":
+            # Less than
+            if is_numeric:
+                if key == "importance":
+                    query["importance_max"] = numeric_value - 0.0001  # Exclusive
+                elif key.startswith("bbox."):
+                    query[f"{key}_max"] = numeric_value - 0.0001
+                elif key == "z_index":
+                    query["z_index_max"] = numeric_value - 0.0001
+            elif key.startswith("attr.") or key.startswith("css."):
+                query[f"{key}_lt"] = value
+        elif op == "<=":
+            # Less than or equal
+            if is_numeric:
+                if key == "importance":
+                    query["importance_max"] = numeric_value
+                elif key.startswith("bbox."):
+                    query[f"{key}_max"] = numeric_value
+                elif key == "z_index":
+                    query["z_index_max"] = numeric_value
+            elif key.startswith("attr.") or key.startswith("css."):
+                query[f"{key}_lte"] = value
+        elif op == "=":
+            # Exact match
+            if key == "role":
+                query["role"] = value
+            elif key == "clickable":
+                query["clickable"] = value.lower() == "true"
+            elif key == "visible":
+                query["visible"] = value.lower() == "true"
+            elif key == "tag":
+                query["tag"] = value
+            elif key == "name" or key == "text":
+                query["text"] = value
+            elif key == "importance" and is_numeric:
+                query["importance"] = numeric_value
+            elif key.startswith("attr."):
+                # Dot notation for attributes: attr.id="submit-btn"
+                attr_key = key[5:]  # Remove "attr." prefix
+                if "attr" not in query:
+                    query["attr"] = {}
+                query["attr"][attr_key] = value
+            elif key.startswith("css."):
+                # Dot notation for CSS: css.color="red"
+                css_key = key[4:]  # Remove "css." prefix
+                if "css" not in query:
+                    query["css"] = {}
+                query["css"][css_key] = value
+
+    return query
+
+
+def match_element(element: Element, query: dict[str, Any]) -> bool:  # noqa: C901
+    """Check if element matches query criteria"""
+
+    # Role exact match
+    if "role" in query:
+        if element.role != query["role"]:
+            return False
+
+    # Role exclusion
+    if "role_exclude" in query:
+        if element.role == query["role_exclude"]:
+            return False
+
+    # Clickable
+    if "clickable" in query:
+        if element.visual_cues.is_clickable != query["clickable"]:
+            return False
+
+    # Visible (using in_viewport and !is_occluded)
+    if "visible" in query:
+        is_visible = element.in_viewport and not element.is_occluded
+        if is_visible != query["visible"]:
+            return False
+
+    # Tag (not yet in Element model, but prepare for future)
+    if "tag" in query:
+        # For now, this will always fail since tag is not in Element model
+        # This is a placeholder for future implementation
+        pass
+
+    # Text exact match
+    if "text" in query:
+        if not element.text or element.text != query["text"]:
+            return False
+
+    # Text contains (case-insensitive)
+    if "text_contains" in query:
+        if not element.text:
+            return False
+        if query["text_contains"].lower() not in element.text.lower():
+            return False
+
+    # Text prefix match
+    if "text_prefix" in query:
+        if not element.text:
+            return False
+        if not element.text.lower().startswith(query["text_prefix"].lower()):
+            return False
+
+    # Text suffix match
+    if "text_suffix" in query:
+        if not element.text:
+            return False
+        if not element.text.lower().endswith(query["text_suffix"].lower()):
+            return False
+
+    # Importance filtering
+    if "importance" in query:
+        if element.importance != query["importance"]:
+            return False
+    if "importance_min" in query:
+        if element.importance < query["importance_min"]:
+            return False
+    if "importance_max" in query:
+        if element.importance > query["importance_max"]:
+            return False
+
+    # BBox filtering (spatial)
+    if "bbox.x_min" in query:
+        if element.bbox.x < query["bbox.x_min"]:
+            return False
+    if "bbox.x_max" in query:
+        if element.bbox.x > query["bbox.x_max"]:
+            return False
+    if "bbox.y_min" in query:
+        if element.bbox.y < query["bbox.y_min"]:
+            return False
+    if "bbox.y_max" in query:
+        if element.bbox.y > query["bbox.y_max"]:
+            return False
+    if "bbox.width_min" in query:
+        if element.bbox.width < query["bbox.width_min"]:
+            return False
+    if "bbox.width_max" in query:
+        if element.bbox.width > query["bbox.width_max"]:
+            return False
+    if "bbox.height_min" in query:
+        if element.bbox.height < query["bbox.height_min"]:
+            return False
+    if "bbox.height_max" in query:
+        if element.bbox.height > query["bbox.height_max"]:
+            return False
+
+    # Z-index filtering
+    if "z_index_min" in query:
+        if element.z_index < query["z_index_min"]:
+            return False
+    if "z_index_max" in query:
+        if element.z_index > query["z_index_max"]:
+            return False
+
+    # In viewport filtering
+    if "in_viewport" in query:
+        if element.in_viewport != query["in_viewport"]:
+            return False
+
+    # Occlusion filtering
+    if "is_occluded" in query:
+        if element.is_occluded != query["is_occluded"]:
+            return False
+
+    # Attribute filtering (dot notation: attr.id="submit-btn")
+    if "attr" in query:
+        # This requires DOM access, which is not available in the Element model
+        # This is a placeholder for future implementation when we add DOM access
+        pass
+
+    # CSS property filtering (dot notation: css.color="red")
+    if "css" in query:
+        # This requires DOM access, which is not available in the Element model
+        # This is a placeholder for future implementation when we add DOM access
+        pass
+
+    return True
+
+
+def query(snapshot: Snapshot, selector: str | dict[str, Any]) -> list[Element]:
+    """
+    Query elements from snapshot using semantic selector
+
+    Args:
+        snapshot: Snapshot object
+        selector: String DSL (e.g., "role=button text~'Sign in'") or dict query
+
+    Returns:
+        List of matching elements, sorted by importance (descending)
+    """
+    # Parse selector if string
+    if isinstance(selector, str):
+        query_dict = parse_selector(selector)
+    else:
+        query_dict = selector
+
+    # Filter elements
+    matches = [el for el in snapshot.elements if match_element(el, query_dict)]
+
+    # Sort by importance (descending)
+    matches.sort(key=lambda el: el.importance, reverse=True)
+
+    return matches
+
+
+def find(snapshot: Snapshot, selector: str | dict[str, Any]) -> Element | None:
+    """
+    Find single element matching selector (best match by importance)
+
+    Args:
+        snapshot: Snapshot object
+        selector: String DSL or dict query
+
+    Returns:
+        Best matching element or None
+    """
+    results = query(snapshot, selector)
+    return results[0] if results else None

sentience/read.py

@@ -0,0 +1,185 @@
+"""
+Read page content - supports raw HTML, text, and markdown formats
+"""
+
+from typing import Literal
+
+from .browser import AsyncSentienceBrowser, SentienceBrowser
+
+
+def read(
+    browser: SentienceBrowser,
+    output_format: Literal["raw", "text", "markdown"] = "raw",
+    enhance_markdown: bool = True,
+) -> dict:
+    """
+    Read page content as raw HTML, text, or markdown
+
+    Args:
+        browser: SentienceBrowser instance
+        output_format: Output format - "raw" (default, returns HTML for external processing),
+                        "text" (plain text), or "markdown" (lightweight or enhanced markdown).
+        enhance_markdown: If True and output_format is "markdown", uses markdownify for better conversion.
+                          If False, uses the extension's lightweight markdown converter.
+
+    Returns:
+        dict with:
+            - status: "success" or "error"
+            - url: Current page URL
+            - format: "raw", "text", or "markdown"
+            - content: Page content as string
+            - length: Content length in characters
+            - error: Error message if status is "error"
+
+    Examples:
+        # Get raw HTML (default) - can be used with markdownify for better conversion
+        result = read(browser)
+        html_content = result["content"]
+
+        # Get high-quality markdown (uses markdownify internally)
+        result = read(browser, output_format="markdown")
+        markdown = result["content"]
+
+        # Get plain text
+        result = read(browser, output_format="text")
+        text = result["content"]
+    """
+    if not browser.page:
+        raise RuntimeError("Browser not started. Call browser.start() first.")
+
+    if output_format == "markdown" and enhance_markdown:
+        # Get raw HTML from the extension first
+        raw_html_result = browser.page.evaluate(
+            """
+            (options) => {
+                return window.sentience.read(options);
+            }
+            """,
+            {"format": "raw"},
+        )
+
+        if raw_html_result.get("status") == "success":
+            html_content = raw_html_result["content"]
+            try:
+                # Use markdownify for enhanced markdown conversion
+                from markdownify import MarkdownifyError, markdownify
+
+                markdown_content = markdownify(html_content, heading_style="ATX", wrap=True)
+                return {
+                    "status": "success",
+                    "url": raw_html_result["url"],
+                    "format": "markdown",
+                    "content": markdown_content,
+                    "length": len(markdown_content),
+                }
+            except ImportError:
+                print(
+                    "Warning: 'markdownify' not installed. Install with 'pip install markdownify' for enhanced markdown. Falling back to extension's markdown."
+                )
+            except MarkdownifyError as e:
+                print(f"Warning: markdownify failed ({e}), falling back to extension's markdown.")
+            except Exception as e:
+                print(
+                    f"Warning: An unexpected error occurred with markdownify ({e}), falling back to extension's markdown."
+                )
+
+    # If not enhanced markdown, or fallback, call extension with requested format
+    result = browser.page.evaluate(
+        """
+        (options) => {
+            return window.sentience.read(options);
+        }
+        """,
+        {"format": output_format},
+    )
+
+    return result
+
+
+async def read_async(
+    browser: AsyncSentienceBrowser,
+    output_format: Literal["raw", "text", "markdown"] = "raw",
+    enhance_markdown: bool = True,
+) -> dict:
+    """
+    Read page content as raw HTML, text, or markdown (async)
+
+    Args:
+        browser: AsyncSentienceBrowser instance
+        output_format: Output format - "raw" (default, returns HTML for external processing),
+                        "text" (plain text), or "markdown" (lightweight or enhanced markdown).
+        enhance_markdown: If True and output_format is "markdown", uses markdownify for better conversion.
+                          If False, uses the extension's lightweight markdown converter.
+
+    Returns:
+        dict with:
+            - status: "success" or "error"
+            - url: Current page URL
+            - format: "raw", "text", or "markdown"
+            - content: Page content as string
+            - length: Content length in characters
+            - error: Error message if status is "error"
+
+    Examples:
+        # Get raw HTML (default) - can be used with markdownify for better conversion
+        result = await read_async(browser)
+        html_content = result["content"]
+
+        # Get high-quality markdown (uses markdownify internally)
+        result = await read_async(browser, output_format="markdown")
+        markdown = result["content"]
+
+        # Get plain text
+        result = await read_async(browser, output_format="text")
+        text = result["content"]
+    """
+    if not browser.page:
+        raise RuntimeError("Browser not started. Call await browser.start() first.")
+
+    if output_format == "markdown" and enhance_markdown:
+        # Get raw HTML from the extension first
+        raw_html_result = await browser.page.evaluate(
+            """
+            (options) => {
+                return window.sentience.read(options);
+            }
+            """,
+            {"format": "raw"},
+        )
+
+        if raw_html_result.get("status") == "success":
+            html_content = raw_html_result["content"]
+            try:
+                # Use markdownify for enhanced markdown conversion
+                from markdownify import MarkdownifyError, markdownify
+
+                markdown_content = markdownify(html_content, heading_style="ATX", wrap=True)
+                return {
+                    "status": "success",
+                    "url": raw_html_result["url"],
+                    "format": "markdown",
+                    "content": markdown_content,
+                    "length": len(markdown_content),
+                }
+            except ImportError:
+                print(
+                    "Warning: 'markdownify' not installed. Install with 'pip install markdownify' for enhanced markdown. Falling back to extension's markdown."
+                )
+            except MarkdownifyError as e:
+                print(f"Warning: markdownify failed ({e}), falling back to extension's markdown.")
+            except Exception as e:
+                print(
+                    f"Warning: An unexpected error occurred with markdownify ({e}), falling back to extension's markdown."
+                )
+
+    # If not enhanced markdown, or fallback, call extension with requested format
+    result = await browser.page.evaluate(
+        """
+        (options) => {
+            return window.sentience.read(options);
+        }
+        """,
+        {"format": output_format},
+    )
+
+    return result