symbolicai 0.21.0__py3-none-any.whl → 1.1.0__py3-none-any.whl

symai/backend/engines/ocr/engine_apilayer.py

@@ -1,11 +1,10 @@
-import requests
 from pathlib import Path
 
-from typing import Optional
+import requests
 
+from ....symbol import Result
 from ...base import Engine
 from ...settings import SYMAI_CONFIG
-from ....symbol import Result
 
 
 class ApiLayerResult(Result):
@@ -14,56 +13,53 @@ class ApiLayerResult(Result):
         self.raw = text
         try:
             dict_ = self._to_symbol(text).ast()
-            self._value = dict_['all_text'] if 'all_text' in dict_ else f'OCR Engine Error: {text} - status code {status_code}'
-        except:
-            self._value = f'OCR Engine Error: {text} - status code {status_code}'
+            self._value = dict_.get(
+                "all_text", f"OCR Engine Error: {text} - status code {status_code}"
+            )
+        except Exception:
+            self._value = f"OCR Engine Error: {text} - status code {status_code}"
 
 
 class OCREngine(Engine):
-    def __init__(self, api_key: Optional[str] = None):
+    def __init__(self, api_key: str | None = None):
         super().__init__()
         # Opening JSON file
         self.config = SYMAI_CONFIG
-        self.headers = {
-            "apikey": self.config['OCR_ENGINE_API_KEY'] if api_key is None else api_key
-        }
+        self.headers = {"apikey": self.config["OCR_ENGINE_API_KEY"] if api_key is None else api_key}
         self.name = self.__class__.__name__
 
     def id(self) -> str:
-        if self.config['OCR_ENGINE_API_KEY']:
-            return 'ocr'
-        return super().id() # default to unregistered
+        if self.config["OCR_ENGINE_API_KEY"]:
+            return "ocr"
+        return super().id()  # default to unregistered
 
     def command(self, *args, **kwargs):
         super().command(*args, **kwargs)
-        if 'OCR_ENGINE_API_KEY' in kwargs:
-            self.headers = {
-                "apikey": kwargs['OCR_ENGINE_API_KEY']
-            }
+        if "OCR_ENGINE_API_KEY" in kwargs:
+            self.headers = {"apikey": kwargs["OCR_ENGINE_API_KEY"]}
 
     def forward(self, argument):
-        kwargs    = argument.kwargs
         image_url = argument.prop.image
 
         if image_url.startswith("file://"):
             file_path = Path(image_url[7:]).resolve()
-            with open(file_path, "rb") as file:
+            with file_path.open("rb") as file:
                 payload = file.read()
-            url      = "https://api.apilayer.com/image_to_text/upload"
+            url = "https://api.apilayer.com/image_to_text/upload"
             response = requests.request("POST", url, headers=self.headers, data=payload)
         else:
-            payload   = {}
-            url      = f"https://api.apilayer.com/image_to_text/url?url={image_url}"
-            response = requests.request("GET", url, headers=self.headers, data = payload)
+            payload = {}
+            url = f"https://api.apilayer.com/image_to_text/url?url={image_url}"
+            response = requests.request("GET", url, headers=self.headers, data=payload)
 
         status_code = response.status_code
-        rsp         = response.text
-        rsp       = ApiLayerResult(response.text, status_code)
-        metadata  = {}
+        rsp = response.text
+        rsp = ApiLayerResult(response.text, status_code)
+        metadata = {}
 
         return [rsp], metadata
 
     def prepare(self, argument):
         assert not argument.prop.processed_input, "OCREngine does not support processed_input."
-        image  = str(argument.prop.image)
+        image = str(argument.prop.image)
         argument.prop.prepared_input = image

symai/backend/engines/output/engine_stdout.py

@@ -7,28 +7,25 @@ class OutputEngine(Engine):
         self.name = self.__class__.__name__
 
     def id(self) -> str:
-        return 'output'
+        return "output"
 
     def forward(self, argument):
-        expr, processed, args, kwargs  = argument.prop.prepared_input
+        expr, processed, args, kwargs = argument.prop.prepared_input
         res = None
         args = [] if args is None else args
         kwargs = {} if kwargs is None else kwargs
         if expr:
-            if processed:
-                res = expr(processed, *args, **kwargs)
-            else:
-                res = expr(*args, **kwargs)
+            res = expr(processed, *args, **kwargs) if processed else expr(*args, **kwargs)
 
         metadata = {}
-        result   = {
-            'result': res,
-            'processed': processed,
-            'args': args,
-            'kwargs': kwargs
-        }
+        result = {"result": res, "processed": processed, "args": args, "kwargs": kwargs}
 
         return [result], metadata
 
     def prepare(self, argument):
-        argument.prop.prepared_input = argument.prop.expr, argument.prop.processed_input, argument.prop.args, argument.prop.kwargs
+        argument.prop.prepared_input = (
+            argument.prop.expr,
+            argument.prop.processed_input,
+            argument.prop.args,
+            argument.prop.kwargs,
+        )

symai/backend/engines/{webscraping → scrape}/engine_requests.py

@@ -10,6 +10,7 @@ service disruption.
 import io
 import logging
 import re
+from typing import Any, ClassVar
 from urllib.parse import parse_qsl, urlencode, urljoin, urlparse, urlunparse
 
 import requests
@@ -19,6 +20,7 @@ from pdfminer.high_level import extract_text
 from requests.structures import CaseInsensitiveDict
 
 from ....symbol import Result
+from ....utils import UserMessage
 from ...base import Engine
 
 logging.getLogger("pdfminer").setLevel(logging.WARNING)
@@ -56,23 +58,23 @@ class RequestsEngine(Engine):
     the requests session stay aligned.
     """
 
-    COMMON_BYPASS_COOKIES = {
+    COMMON_BYPASS_COOKIES: ClassVar[dict[str, str]] = {
         # Some forums display consent or age gates once if a friendly cookie is set.
         "cookieconsent_status": "allow",
         "accepted_cookies": "yes",
         "age_verified": "1",
     }
 
-    DEFAULT_HEADERS = {
+    DEFAULT_HEADERS: ClassVar[dict[str, str]] = {
         "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
-                      "AppleWebKit/537.36 (KHTML, like Gecko) "
-                      "Chrome/120.0.0.0 Safari/537.36",
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/120.0.0.0 Safari/537.36",
         "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
         "Accept-Language": "en-US,en;q=0.9",
         "DNT": "1",
     }
 
-    _SAMESITE_CANONICAL = {
+    _SAMESITE_CANONICAL: ClassVar[dict[str, str]] = {
         "strict": "Strict",
         "lax": "Lax",
         "none": "None",
@@ -156,6 +158,58 @@ class RequestsEngine(Engine):
             payload["sameSite"] = same_site
         return payload
 
+    def _collect_playwright_cookies(self, hostname: str) -> list[dict[str, Any]]:
+        if not hostname:
+            return []
+        cookie_payload = []
+        for cookie in self.session.cookies:
+            payload = self._playwright_cookie_payload(cookie, hostname)
+            if payload:
+                cookie_payload.append(payload)
+        return cookie_payload
+
+    @staticmethod
+    def _add_cookies_to_context(context, cookie_payload: list[dict[str, Any]]) -> None:
+        if cookie_payload:
+            context.add_cookies(cookie_payload)
+
+    @staticmethod
+    def _navigate_playwright_page(
+        page, url: str, wait_selector: str | None, wait_until: str, timeout_ms: int, timeout_error
+    ):
+        try:
+            response = page.goto(url, wait_until=wait_until, timeout=timeout_ms)
+            if wait_selector:
+                page.wait_for_selector(wait_selector, timeout=timeout_ms)
+            return response, None
+        except timeout_error as exc:
+            return None, exc
+
+    @staticmethod
+    def _safe_page_content(page) -> str:
+        try:
+            return page.content()
+        except Exception:
+            return ""
+
+    def _sync_cookies_from_context(self, context) -> None:
+        for cookie in context.cookies():
+            self.session.cookies.set(
+                cookie["name"],
+                cookie["value"],
+                domain=cookie.get("domain"),
+                path=cookie.get("path", "/"),
+            )
+
+    @staticmethod
+    def _rendered_response_metadata(page, response):
+        final_url = page.url
+        status = response.status if response is not None else 200
+        headers = CaseInsensitiveDict(response.headers if response is not None else {})
+        if "content-type" not in headers:
+            headers["Content-Type"] = "text/html; charset=utf-8"
+        return final_url, status, headers
+
     def _follow_meta_refresh(self, resp, timeout=15):
         """
         Some old forums use <meta http-equiv="refresh" content="0;url=...">
@@ -180,19 +234,28 @@ class RequestsEngine(Engine):
             return resp
         return self.session.get(target, timeout=timeout, allow_redirects=True)
 
-    def _fetch_with_playwright(self, url: str, wait_selector: str = None, wait_until: str = "networkidle", timeout: float = None):
+    def _fetch_with_playwright(
+        self,
+        url: str,
+        wait_selector: str | None = None,
+        wait_until: str = "networkidle",
+        timeout: float | None = None,
+    ):
         """
         Render the target URL in a headless browser to execute JavaScript and
         return a synthetic ``requests.Response`` object to keep downstream
         processing consistent with the non-JS path.
         """
         try:
-            from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeoutError
+            # Playwright is optional; import only when JS rendering is requested.
+            from playwright.sync_api import TimeoutError as PlaywrightTimeoutError  # noqa
+            from playwright.sync_api import sync_playwright  # noqa
+
             logging.getLogger("playwright").setLevel(logging.WARNING)
         except ImportError as exc:
-            raise RuntimeError(
-                "Playwright is not installed. Install symbolicai[webscraping] with Playwright extras to enable render_js."
-            ) from exc
+            msg = "Playwright is not installed. Install symbolicai[scrape] with Playwright extras to enable render_js."
+            UserMessage(msg)
+            raise RuntimeError(msg) from exc
 
         timeout_seconds = timeout if timeout is not None else self.timeout
         timeout_ms = max(int(timeout_seconds * 1000), 0)
@@ -200,12 +263,7 @@ class RequestsEngine(Engine):
 
         parsed = urlparse(url)
         hostname = parsed.hostname or ""
-        cookie_payload = []
-        if hostname:
-            for cookie in self.session.cookies:
-                payload = self._playwright_cookie_payload(cookie, hostname)
-                if payload:
-                    cookie_payload.append(payload)
+        cookie_payload = self._collect_playwright_cookies(hostname)
 
         content = ""
         final_url = url
@@ -219,42 +277,26 @@ class RequestsEngine(Engine):
                 java_script_enabled=True,
                 ignore_https_errors=not self.verify_ssl,
             )
-            if cookie_payload:
-                context.add_cookies(cookie_payload)
-            page = context.new_page()
-
-            navigation_error = None
-            response = None
             try:
-                try:
-                    response = page.goto(url, wait_until=wait_until, timeout=timeout_ms)
-                    if wait_selector:
-                        page.wait_for_selector(wait_selector, timeout=timeout_ms)
-                except PlaywrightTimeoutError as exc:
-                    navigation_error = exc
-
-                try:
-                    content = page.content()
-                except Exception:
-                    content = ""
-
-                # Always persist Playwright cookies back into the requests session.
-                for cookie in context.cookies():
-                    self.session.cookies.set(
-                        cookie["name"],
-                        cookie["value"],
-                        domain=cookie.get("domain"),
-                        path=cookie.get("path", "/"),
-                    )
-
-                final_url = page.url
-                status = response.status if response is not None else 200
-                headers = CaseInsensitiveDict(response.headers if response is not None else {})
-                if "content-type" not in headers:
-                    headers["Content-Type"] = "text/html; charset=utf-8"
-
+                self._add_cookies_to_context(context, cookie_payload)
+                page = context.new_page()
+
+                response, navigation_error = self._navigate_playwright_page(
+                    page,
+                    url,
+                    wait_selector,
+                    wait_until,
+                    timeout_ms,
+                    PlaywrightTimeoutError,
+                )
+                content = self._safe_page_content(page)
+                self._sync_cookies_from_context(context)
+
+                final_url, status, headers = self._rendered_response_metadata(page, response)
                 if navigation_error and not content:
-                    raise requests.exceptions.Timeout(f"Playwright timed out while rendering {url}") from navigation_error
+                    msg = f"Playwright timed out while rendering {url}"
+                    UserMessage(msg)
+                    raise requests.exceptions.Timeout(msg) from navigation_error
             finally:
                 context.close()
                 browser.close()
@@ -268,7 +310,7 @@ class RequestsEngine(Engine):
         return rendered_response
 
     def id(self) -> str:
-        return 'webscraping'
+        return "scrape"
 
     def forward(self, argument):
         """
@@ -284,8 +326,11 @@ class RequestsEngine(Engine):
         self._maybe_set_bypass_cookies(url)
 
         parsed = urlparse(url)
-        qs = [(k, v) for k, v in parse_qsl(parsed.query, keep_blank_values=True)
-              if k.lower() not in {"utm_source", "utm_medium", "utm_campaign"}]
+        qs = [
+            (k, v)
+            for k, v in parse_qsl(parsed.query, keep_blank_values=True)
+            if k.lower() not in {"utm_source", "utm_medium", "utm_campaign"}
+        ]
         clean_url = urlunparse(parsed._replace(query=urlencode(qs)))
 
         render_js = kwargs.get("render_js")
@@ -302,7 +347,9 @@ class RequestsEngine(Engine):
                 timeout=render_timeout,
             )
         else:
-            resp = self.session.get(clean_url, timeout=self.timeout, allow_redirects=True, verify=self.verify_ssl)
+            resp = self.session.get(
+                clean_url, timeout=self.timeout, allow_redirects=True, verify=self.verify_ssl
+            )
         resp.raise_for_status()
 
         # Follow a legacy meta refresh once (do AFTER normal HTTP redirects)