symbolicai 0.19.0__tar.gz → 0.20.1__tar.gz

{symbolicai-0.19.0/docs/source → symbolicai-0.20.1}/LICENSE

@@ -1,24 +1,20 @@
-License
-=======
-
 BSD 3-Clause License
 
-Copyright (c) 2024, ExtensityAI FlexCo.
-All rights reserved.
+Copyright (c) 2025, ExtensityAI FlexCo
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are met:
 
-* Redistributions of source code must retain the above copyright notice, this
-  list of conditions and the following disclaimer.
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
 
-* Redistributions in binary form must reproduce the above copyright notice,
-  this list of conditions and the following disclaimer in the documentation
-  and/or other materials provided with the distribution.
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
 
-* Neither the name of the copyright holder nor the names of its
-  contributors may be used to endorse or promote products derived from
-  this software without specific prior written permission.
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
 
 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

{symbolicai-0.19.0 → symbolicai-0.20.1}/PKG-INFO

@@ -1,8 +1,37 @@
 Metadata-Version: 2.4
 Name: symbolicai
-Version: 0.19.0
+Version: 0.20.1
 Summary: A Neurosymbolic Perspective on Large Language Models
 Author-email: Marius-Constantin Dinu <marius@extensity.ai>, Leoveanu-Condrei Claudiu <leo@extensity.ai>
+License: BSD 3-Clause License
+        
+        Copyright (c) 2025, ExtensityAI FlexCo
+        
+        Redistribution and use in source and binary forms, with or without
+        modification, are permitted provided that the following conditions are met:
+        
+        1. Redistributions of source code must retain the above copyright notice, this
+           list of conditions and the following disclaimer.
+        
+        2. Redistributions in binary form must reproduce the above copyright notice,
+           this list of conditions and the following disclaimer in the documentation
+           and/or other materials provided with the distribution.
+        
+        3. Neither the name of the copyright holder nor the names of its
+           contributors may be used to endorse or promote products derived from
+           this software without specific prior written permission.
+        
+        THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+        AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+        IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+        DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+        FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+        DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+        SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+        CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+        OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+        OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+        
 Project-URL: Homepage, https://extensity.ai
 Project-URL: GitHub, https://github.com/ExtensityAI/symbolicai
 Keywords: probabilistic programming,machine learning
@@ -11,6 +40,7 @@ Classifier: License :: OSI Approved :: BSD License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
+License-File: LICENSE
 Requires-Dist: attrs>=23.2.0
 Requires-Dist: setuptools>=70.0.0
 Requires-Dist: toml>=0.10.2
@@ -53,6 +83,7 @@ Requires-Dist: pycryptodome>=3.20.0
 Requires-Dist: httpx>=0.27.2
 Requires-Dist: nest-asyncio>=1.6.0
 Requires-Dist: rich>=13.9.4
+Requires-Dist: playwright>=1.55.0
 Provides-Extra: bitsandbytes
 Requires-Dist: bitsandbytes>=0.43.1; extra == "bitsandbytes"
 Provides-Extra: blip2
@@ -69,6 +100,7 @@ Provides-Extra: webscraping
 Requires-Dist: beautifulsoup4>=4.12.3; extra == "webscraping"
 Requires-Dist: trafilatura>=2.0.0; extra == "webscraping"
 Requires-Dist: pdfminer.six; extra == "webscraping"
+Requires-Dist: playwright>=1.45.0; extra == "webscraping"
 Provides-Extra: llama-cpp
 Requires-Dist: llama-cpp-python[server]>=0.3.7; extra == "llama-cpp"
 Provides-Extra: wolframalpha
@@ -92,6 +124,7 @@ Requires-Dist: symbolicai[webscraping]; extra == "all"
 Requires-Dist: symbolicai[serpapi]; extra == "all"
 Requires-Dist: symbolicai[services]; extra == "all"
 Requires-Dist: symbolicai[solver]; extra == "all"
+Dynamic: license-file
 
 # **SymbolicAI: A neuro-symbolic perspective on LLMs**
 <img src="https://raw.githubusercontent.com/ExtensityAI/symbolicai/refs/heads/main/assets/images/banner.png">
@@ -420,7 +453,7 @@ Now, there are tools like DeepWiki that provide better documentation than we cou
 
 ## 📝 License
 
-This project is licensed under the BSD-3-Clause License - refer to [the docs](https://symbolicai.readthedocs.io/en/latest/LICENSE.html).
+This project is licensed under the BSD-3-Clause License.
 
 ## Like this Project?
 

{symbolicai-0.19.0 → symbolicai-0.20.1}/README.md

@@ -325,7 +325,7 @@ Now, there are tools like DeepWiki that provide better documentation than we cou
 
 ## 📝 License
 
-This project is licensed under the BSD-3-Clause License - refer to [the docs](https://symbolicai.readthedocs.io/en/latest/LICENSE.html).
+This project is licensed under the BSD-3-Clause License.
 
 ## Like this Project?
 

{symbolicai-0.19.0 → symbolicai-0.20.1}/docs/source/ENGINES/search_engine.md

@@ -78,7 +78,7 @@ res = search("What local events are happening today?",
              })
 
 # Control the amount of search context
-res = search("Explain quantum computing developments", search_context_size="high")
+res = search("Explain quantum computing developments")
 ```
 
 Here's how to configure the OpenAI search engine:

{symbolicai-0.19.0 → symbolicai-0.20.1}/docs/source/SUMMARY.md

@@ -36,6 +36,3 @@
 * [Chatbot CLI](TOOLS/chatbot.md)
 * [Package Manager](TOOLS/packages.md)
 * [Shell](TOOLS/shell.md)
-
-## License
-* [License](LICENSE)

{symbolicai-0.19.0 → symbolicai-0.20.1}/pyproject.toml

@@ -64,14 +64,15 @@ dependencies = [
     "pycryptodome>=3.20.0",
     "httpx>=0.27.2",
     "nest-asyncio>=1.6.0",
-    "rich>=13.9.4"
+    "rich>=13.9.4",
+    "playwright>=1.55.0",
 ]
 
 [project.optional-dependencies]
 bitsandbytes = ["bitsandbytes>=0.43.1"] # handle separately because of Apple Silicon
 blip2        = ["decord>=0.6.0", "salesforce-lavis>=1.0.0", "opencv-python-headless>=4.5.5.64"]
 hf           = ["transformers>=4.45.2", "accelerate>=0.33.0", "peft>=0.13.1", "datasets>=3.0.1", "trl>=0.11.3"]
-webscraping  = ["beautifulsoup4>=4.12.3", "trafilatura>=2.0.0", "pdfminer.six"]
+webscraping  = ["beautifulsoup4>=4.12.3", "trafilatura>=2.0.0", "pdfminer.six", "playwright>=1.45.0"]
 llama_cpp    = ["llama-cpp-python[server]>=0.3.7"] # handle separately since this dependency may not compile and require special maintenance
 wolframalpha = ["wolframalpha>=5.0.0"]
 whisper      = ["openai-whisper>=20240930", "numba>=0.60.0"]

{symbolicai-0.19.0 → symbolicai-0.20.1}/symai/__init__.py

@@ -33,7 +33,7 @@ os.environ['TOKENIZERS_PARALLELISM'] = "false"
 # Create singleton instance
 config_manager = settings.SymAIConfig()
 
-SYMAI_VERSION = "0.19.0"
+SYMAI_VERSION = "0.20.1"
 __version__   = SYMAI_VERSION
 __root_dir__  = config_manager.config_dir
 

{symbolicai-0.19.0 → symbolicai-0.20.1}/symai/backend/engines/neurosymbolic/engine_anthropic_claudeX_chat.py

@@ -56,7 +56,8 @@ class ClaudeXChatEngine(Engine, AnthropicMixin):
            self.config.get('NEUROSYMBOLIC_ENGINE_MODEL').startswith('claude') and \
            ('3-7' not in self.config.get('NEUROSYMBOLIC_ENGINE_MODEL') and \
             '4-0' not in self.config.get('NEUROSYMBOLIC_ENGINE_MODEL') and \
-            '4-1' not in self.config.get('NEUROSYMBOLIC_ENGINE_MODEL')):
+            '4-1' not in self.config.get('NEUROSYMBOLIC_ENGINE_MODEL') and \
+            '4-5' not in self.config.get('NEUROSYMBOLIC_ENGINE_MODEL')):
                return 'neurosymbolic'
         return super().id() # default to unregistered
 

{symbolicai-0.19.0 → symbolicai-0.20.1}/symai/backend/engines/neurosymbolic/engine_anthropic_claudeX_reasoning.py

@@ -57,7 +57,8 @@ class ClaudeXReasoningEngine(Engine, AnthropicMixin):
            self.config.get('NEUROSYMBOLIC_ENGINE_MODEL').startswith('claude') and \
            ('3-7' in self.config.get('NEUROSYMBOLIC_ENGINE_MODEL') or \
             '4-0' in self.config.get('NEUROSYMBOLIC_ENGINE_MODEL') or \
-            '4-1' in self.config.get('NEUROSYMBOLIC_ENGINE_MODEL')):
+            '4-1' in self.config.get('NEUROSYMBOLIC_ENGINE_MODEL') or \
+            '4-5' in self.config.get('NEUROSYMBOLIC_ENGINE_MODEL')):
                return 'neurosymbolic'
         return super().id() # default to unregistered
 

{symbolicai-0.19.0 → symbolicai-0.20.1}/symai/backend/engines/search/engine_openai.py

@@ -230,6 +230,8 @@ class SearchResult(Result):
 
 
 class GPTXSearchEngine(Engine):
+    MAX_ALLOWED_DOMAINS = 20
+
     def __init__(self, api_key: str | None = None, model: str | None = None):
         super().__init__()
         self.config = deepcopy(SYMAI_CONFIG)
@@ -250,6 +252,65 @@ class GPTXSearchEngine(Engine):
             return 'search'
         return super().id()  # default to unregistered
 
+    def _extract_netloc(self, raw_domain: str | None) -> str | None:
+        if not isinstance(raw_domain, str):
+            return None
+        candidate = raw_domain.strip()
+        if not candidate:
+            return None
+        parsed = urlsplit(candidate if '://' in candidate else f"//{candidate}")
+        netloc = parsed.netloc or parsed.path
+        if not netloc:
+            return None
+        if '@' in netloc:
+            netloc = netloc.split('@', 1)[1]
+        if ':' in netloc:
+            netloc = netloc.split(':', 1)[0]
+        netloc = netloc.strip('.').strip()
+        if not netloc:
+            return None
+        return netloc.lower()
+
+    def _normalize_allowed_domains(self, domains: list[str] | None) -> list[str]:
+        if not domains or not isinstance(domains, list):
+            return []
+
+        normalized: list[str] = []
+        seen: set[str] = set()
+        for domain in domains:
+            netloc = self._extract_netloc(domain)
+            if not netloc or netloc in seen:
+                continue
+            # Validate that netloc is actually a valid domain
+            if not self._is_domain(netloc):
+                continue
+            normalized.append(netloc)
+            seen.add(netloc)
+            if len(normalized) >= self.MAX_ALLOWED_DOMAINS:
+                break
+        return normalized
+
+    def _is_domain(self, s: str) -> bool:
+        _label_re = re.compile(r"^[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?$")
+        if not s:
+            return False
+        host = s.strip().rstrip(".")
+        # If the input might be a URL, extract the hostname via urllib:
+        if "://" in host or "/" in host or "@" in host:
+            host = urlsplit(host if "://" in host else f"//{host}").hostname or ""
+        if not host:
+            return False
+        try:
+            host_ascii = host.encode("idna").decode("ascii")
+        except Exception:
+            return False
+        if len(host_ascii) > 253:
+            return False
+        labels = host_ascii.split(".")
+        if len(labels) < 2:  # require a dot (reject "google")
+            return False
+        return all(_label_re.fullmatch(lbl or "") for lbl in labels)
+
     def command(self, *args, **kwargs):
         super().command(*args, **kwargs)
         if 'SEARCH_ENGINE_API_KEY' in kwargs:
@@ -261,20 +322,23 @@ class GPTXSearchEngine(Engine):
         messages = argument.prop.prepared_input
         kwargs = argument.kwargs
 
-        tool_definition = {"type": "web_search_preview"}
+        tool_definition = {"type": "web_search"}
         user_location = kwargs.get('user_location')
         if user_location:
             tool_definition['user_location'] = user_location
-        search_context_size = kwargs.get('search_context_size')
-        if search_context_size:
-            tool_definition['search_context_size'] = search_context_size
+
+        allowed_domains = self._normalize_allowed_domains(kwargs.get('allowed_domains'))
+        if allowed_domains:
+            tool_definition['filters'] = {
+                'allowed_domains': allowed_domains
+            }
 
         self.model = kwargs.get('model', self.model) # Important for MetadataTracker to work correctly
         payload = {
             "model": self.model,
             "input": messages,
             "tools": [tool_definition],
-            "tool_choice": {"type": "web_search_preview"} if self.model not in OPENAI_REASONING_MODELS else "auto" # force the use of web search tool for non-reasoning models
+            "tool_choice": {"type": "web_search"} if self.model not in OPENAI_REASONING_MODELS else "auto" # force the use of web search tool for non-reasoning models
         }
 
         try:

symbolicai-0.20.1/symai/backend/engines/webscraping/engine_requests.py

@@ -0,0 +1,323 @@
+"""
+WARNING: This module implements a naive web scraping engine meant for light
+testing. It does not prevent IP bans, bot detection, or terms-of-service
+violations. Use only where scraping is legally permitted and respect each
+site's robots directives. For production workloads, add robust rate limiting,
+consent handling, rotating proxies/VPNs, and ongoing monitoring to avoid
+service disruption.
+"""
+
+import io
+import logging
+import re
+from urllib.parse import parse_qsl, urlencode, urljoin, urlparse, urlunparse
+
+import requests
+import trafilatura
+from bs4 import BeautifulSoup
+from pdfminer.high_level import extract_text
+from requests.structures import CaseInsensitiveDict
+
+from ....symbol import Result
+from ...base import Engine
+
+logging.getLogger("pdfminer").setLevel(logging.WARNING)
+logging.getLogger("trafilatura").setLevel(logging.WARNING)
+
+
+class RequestsResult(Result):
+    def __init__(self, value, output_format="markdown", **kwargs) -> None:
+        super().__init__(value, **kwargs)
+        self.output_format = output_format
+        self.raw = value
+        self._value = self.extract()
+
+    def extract(self):
+        ctype = self.raw.headers.get("Content-Type", "").lower()
+        is_pdf = "application/pdf" in ctype or self.raw.url.lower().endswith(".pdf")
+        try:
+            if is_pdf:
+                with io.BytesIO(self.raw.content) as fh:
+                    self._value = extract_text(fh)
+            else:
+                decoded = trafilatura.load_html(self.raw.content)
+                self._value = trafilatura.extract(decoded, output_format=self.output_format)
+        except Exception:  # keep broad except to avoid hard failures
+            self._value = None
+        return self._value
+
+
+class RequestsEngine(Engine):
+    """
+    Lightweight HTTP/Playwright fetching pipeline for content extraction.
+
+    The engine favors clarity over stealth. Helper methods normalize cookie
+    metadata before handing it to Playwright so that the headless browser and
+    the requests session stay aligned.
+    """
+
+    COMMON_BYPASS_COOKIES = {
+        # Some forums display consent or age gates once if a friendly cookie is set.
+        "cookieconsent_status": "allow",
+        "accepted_cookies": "yes",
+        "age_verified": "1",
+    }
+
+    DEFAULT_HEADERS = {
+        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+                      "AppleWebKit/537.36 (KHTML, like Gecko) "
+                      "Chrome/120.0.0.0 Safari/537.36",
+        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.9",
+        "DNT": "1",
+    }
+
+    _SAMESITE_CANONICAL = {
+        "strict": "Strict",
+        "lax": "Lax",
+        "none": "None",
+    }
+
+    def __init__(self, timeout=15, verify_ssl=True, user_agent=None):
+        """
+        Args:
+            timeout: Seconds to wait for network operations before aborting.
+            verify_ssl: Toggle for TLS certificate verification.
+            user_agent: Optional override for the default desktop Chrome UA.
+        """
+        super().__init__()
+        self.timeout = timeout
+        self.verify_ssl = verify_ssl
+        self.name = self.__class__.__name__
+
+        headers = dict(self.DEFAULT_HEADERS)
+        if user_agent:
+            headers["User-Agent"] = user_agent
+
+        self.session = requests.Session()
+        self.session.headers.update(headers)
+
+    def _maybe_set_bypass_cookies(self, url: str):
+        netloc = urlparse(url).hostname
+        if not netloc:
+            return
+        for k, v in self.COMMON_BYPASS_COOKIES.items():
+            self.session.cookies.set(k, v, domain=netloc)
+
+    @staticmethod
+    def _normalize_http_only(raw_value, key_present):
+        """
+        Playwright expects a boolean. Cookie metadata can arrive as strings,
+        numbers, or placeholder objects, so normalize defensively.
+        """
+        if isinstance(raw_value, bool):
+            return raw_value
+        if isinstance(raw_value, str):
+            normalized = raw_value.strip().lower()
+            if normalized in {"false", "0", "no"}:
+                return False
+            if normalized in {"true", "1", "yes"}:
+                return True
+        if raw_value is None:
+            return key_present
+        return bool(raw_value)
+
+    @classmethod
+    def _normalize_same_site(cls, raw_value):
+        if raw_value is None:
+            return None
+        normalized = str(raw_value).strip().lower()
+        return cls._SAMESITE_CANONICAL.get(normalized)
+
+    def _playwright_cookie_payload(self, cookie, hostname):
+        """
+        Convert a requests cookie into Playwright-friendly format or return None
+        if the cookie does not apply to the hostname.
+        """
+        domain = (cookie.domain or hostname).lstrip(".")
+        if not hostname.endswith(domain):
+            return None
+
+        rest_attrs = {k.lower(): v for k, v in cookie._rest.items()}
+        http_only = self._normalize_http_only(rest_attrs.get("httponly"), "httponly" in rest_attrs)
+        payload = {
+            "name": cookie.name,
+            "value": cookie.value,
+            "domain": cookie.domain or hostname,
+            "path": cookie.path or "/",
+            "httpOnly": http_only,
+            "secure": cookie.secure,
+        }
+        if cookie.expires:
+            payload["expires"] = cookie.expires
+
+        same_site = self._normalize_same_site(rest_attrs.get("samesite"))
+        if same_site:
+            payload["sameSite"] = same_site
+        return payload
+
+    def _follow_meta_refresh(self, resp, timeout=15):
+        """
+        Some old forums use <meta http-equiv="refresh" content="0;url=...">
+        (sometimes to simulate a popup or interstitial). Follow it once.
+        """
+        ctype = resp.headers.get("Content-Type", "")
+        if "text/html" not in ctype.lower():
+            return resp
+        # Use apparent encoding to decode legacy charsets
+        soup = BeautifulSoup(resp.text, "html.parser")
+        resp.encoding = resp.encoding or resp.apparent_encoding
+        meta = soup.find("meta", attrs={"http-equiv": re.compile("^refresh$", re.I)})
+        if not meta or "content" not in meta.attrs:
+            return resp
+        m = re.search(r"url=(.+)", meta["content"], flags=re.I)
+        if not m:
+            return resp
+        refresh_url = m.group(1).strip().strip("'\"")
+        target = urljoin(resp.url, refresh_url)
+        # Avoid loops
+        if target == resp.url:
+            return resp
+        return self.session.get(target, timeout=timeout, allow_redirects=True)
+
+    def _fetch_with_playwright(self, url: str, wait_selector: str = None, wait_until: str = "networkidle", timeout: float = None):
+        """
+        Render the target URL in a headless browser to execute JavaScript and
+        return a synthetic ``requests.Response`` object to keep downstream
+        processing consistent with the non-JS path.
+        """
+        try:
+            from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeoutError
+            logging.getLogger("playwright").setLevel(logging.WARNING)
+        except ImportError as exc:
+            raise RuntimeError(
+                "Playwright is not installed. Install symbolicai[webscraping] with Playwright extras to enable render_js."
+            ) from exc
+
+        timeout_seconds = timeout if timeout is not None else self.timeout
+        timeout_ms = max(int(timeout_seconds * 1000), 0)
+        user_agent = self.session.headers.get("User-Agent")
+
+        parsed = urlparse(url)
+        hostname = parsed.hostname or ""
+        cookie_payload = []
+        if hostname:
+            for cookie in self.session.cookies:
+                payload = self._playwright_cookie_payload(cookie, hostname)
+                if payload:
+                    cookie_payload.append(payload)
+
+        content = ""
+        final_url = url
+        status = 200
+        headers = CaseInsensitiveDict()
+
+        with sync_playwright() as playwright:
+            browser = playwright.chromium.launch(headless=True)
+            context = browser.new_context(
+                user_agent=user_agent,
+                java_script_enabled=True,
+                ignore_https_errors=not self.verify_ssl,
+            )
+            if cookie_payload:
+                context.add_cookies(cookie_payload)
+            page = context.new_page()
+
+            navigation_error = None
+            response = None
+            try:
+                try:
+                    response = page.goto(url, wait_until=wait_until, timeout=timeout_ms)
+                    if wait_selector:
+                        page.wait_for_selector(wait_selector, timeout=timeout_ms)
+                except PlaywrightTimeoutError as exc:
+                    navigation_error = exc
+
+                try:
+                    content = page.content()
+                except Exception:
+                    content = ""
+
+                # Always persist Playwright cookies back into the requests session.
+                for cookie in context.cookies():
+                    self.session.cookies.set(
+                        cookie["name"],
+                        cookie["value"],
+                        domain=cookie.get("domain"),
+                        path=cookie.get("path", "/"),
+                    )
+
+                final_url = page.url
+                status = response.status if response is not None else 200
+                headers = CaseInsensitiveDict(response.headers if response is not None else {})
+                if "content-type" not in headers:
+                    headers["Content-Type"] = "text/html; charset=utf-8"
+
+                if navigation_error and not content:
+                    raise requests.exceptions.Timeout(f"Playwright timed out while rendering {url}") from navigation_error
+            finally:
+                context.close()
+                browser.close()
+
+        rendered_response = requests.Response()
+        rendered_response.status_code = status
+        rendered_response._content = content.encode("utf-8", errors="replace")
+        rendered_response.url = final_url
+        rendered_response.headers = headers
+        rendered_response.encoding = "utf-8"
+        return rendered_response
+
+    def id(self) -> str:
+        return 'webscraping'
+
+    def forward(self, argument):
+        """
+        Return raw bytes of the final page body.
+        - Retries network errors (not programming bugs).
+        - Handles legacy redirects via meta refresh.
+        - Attempts to bypass simple consent/age popups by pre-seeding cookies.
+        """
+        url = argument.prop.prepared_input
+        kwargs = argument.kwargs
+        output_format = kwargs.get("output_format", "markdown")
+
+        self._maybe_set_bypass_cookies(url)
+
+        parsed = urlparse(url)
+        qs = [(k, v) for k, v in parse_qsl(parsed.query, keep_blank_values=True)
+              if k.lower() not in {"utm_source", "utm_medium", "utm_campaign"}]
+        clean_url = urlunparse(parsed._replace(query=urlencode(qs)))
+
+        render_js = kwargs.get("render_js")
+        render_wait_selector = kwargs.get("render_wait_selector")
+        render_wait_until = kwargs.get("render_wait_until", "networkidle")
+        render_timeout = kwargs.get("render_timeout")
+
+        # Prefer fast requests path unless the caller opts into JS rendering.
+        if render_js:
+            resp = self._fetch_with_playwright(
+                clean_url,
+                wait_selector=render_wait_selector,
+                wait_until=render_wait_until,
+                timeout=render_timeout,
+            )
+        else:
+            resp = self.session.get(clean_url, timeout=self.timeout, allow_redirects=True, verify=self.verify_ssl)
+        resp.raise_for_status()
+
+        # Follow a legacy meta refresh once (do AFTER normal HTTP redirects)
+        resp2 = self._follow_meta_refresh(resp, timeout=self.timeout)
+        if resp2 is not resp:
+            resp2.raise_for_status()
+            resp = resp2
+
+        metadata = {
+            "response_source": "playwright" if render_js else "requests",
+            "render_js": bool(render_js),
+            "final_url": resp.url,
+        }
+        result = RequestsResult(resp, output_format)
+        return [result], metadata
+
+    def prepare(self, argument):
+        argument.prop.prepared_input = str(argument.prop.url)

{symbolicai-0.19.0 → symbolicai-0.20.1}/symai/backend/mixin/anthropic.py

@@ -14,6 +14,8 @@ SUPPORTED_REASONING_MODELS = [
     "claude-opus-4-0",
     "claude-sonnet-4-0",
     'claude-3-7-sonnet-latest',
+    'claude-haiku-4-5',
+    'claude-sonnet-4-5',
 ]
 
 class AnthropicMixin:
@@ -22,6 +24,8 @@ class AnthropicMixin:
            self.model == 'claude-opus-4-0' or \
            self.model == 'claude-sonnet-4-0' or \
            self.model == 'claude-3-7-sonnet-latest' or \
+           self.model == 'claude-haiku-4-5' or \
+           self.model == 'claude-sonnet-4-5' or \
            self.model == 'claude-3-5-sonnet-latest' or \
            self.model == 'claude-3-5-sonnet-20241022' or \
            self.model == 'claude-3-5-sonnet-20240620' or \
@@ -33,7 +37,9 @@ class AnthropicMixin:
 
     def api_max_response_tokens(self):
         if self.model == 'claude-sonnet-4-0' or \
-           self.model == 'claude-3-7-sonnet-latest':
+           self.model == 'claude-3-7-sonnet-latest' or \
+           self.model == 'claude-haiku-4-5' or \
+           self.model == 'claude-sonnet-4-5':
             return 64_000
         if self.model == 'claude-opus-4-1' or \
            self.model == 'claude-opus-4-0':

{symbolicai-0.19.0 → symbolicai-0.20.1}/symai/extended/interfaces/naive_webscraping.py

@@ -10,6 +10,8 @@ class naive_webscraping(Expression):
 
     def __call__(self, url: str, **kwargs) -> RequestsResult:
         @core.scrape(url=url, **kwargs)
-        def _func(_) -> RequestsResult:
-            pass
+        def _func(_, *args, **inner_kwargs) -> RequestsResult:
+            # The fallback path may inject debugging kwargs like `error`/`stack_trace`;
+            # accept and ignore them so EngineRepository can surface structured failures.
+            return None
         return _func(self)