symbolicai 1.0.0__py3-none-any.whl → 1.1.0__py3-none-any.whl

symai/backend/engines/ocr/engine_apilayer.py

@@ -13,9 +13,11 @@ class ApiLayerResult(Result):
         self.raw = text
         try:
             dict_ = self._to_symbol(text).ast()
-            self._value = dict_.get('all_text', f'OCR Engine Error: {text} - status code {status_code}')
+            self._value = dict_.get(
+                "all_text", f"OCR Engine Error: {text} - status code {status_code}"
+            )
         except Exception:
-            self._value = f'OCR Engine Error: {text} - status code {status_code}'
+            self._value = f"OCR Engine Error: {text} - status code {status_code}"
 
 
 class OCREngine(Engine):
@@ -23,22 +25,18 @@ class OCREngine(Engine):
         super().__init__()
         # Opening JSON file
         self.config = SYMAI_CONFIG
-        self.headers = {
-            "apikey": self.config['OCR_ENGINE_API_KEY'] if api_key is None else api_key
-        }
+        self.headers = {"apikey": self.config["OCR_ENGINE_API_KEY"] if api_key is None else api_key}
         self.name = self.__class__.__name__
 
     def id(self) -> str:
-        if self.config['OCR_ENGINE_API_KEY']:
-            return 'ocr'
-        return super().id() # default to unregistered
+        if self.config["OCR_ENGINE_API_KEY"]:
+            return "ocr"
+        return super().id()  # default to unregistered
 
     def command(self, *args, **kwargs):
         super().command(*args, **kwargs)
-        if 'OCR_ENGINE_API_KEY' in kwargs:
-            self.headers = {
-                "apikey": kwargs['OCR_ENGINE_API_KEY']
-            }
+        if "OCR_ENGINE_API_KEY" in kwargs:
+            self.headers = {"apikey": kwargs["OCR_ENGINE_API_KEY"]}
 
     def forward(self, argument):
         image_url = argument.prop.image
@@ -47,21 +45,21 @@ class OCREngine(Engine):
             file_path = Path(image_url[7:]).resolve()
             with file_path.open("rb") as file:
                 payload = file.read()
-            url      = "https://api.apilayer.com/image_to_text/upload"
+            url = "https://api.apilayer.com/image_to_text/upload"
             response = requests.request("POST", url, headers=self.headers, data=payload)
         else:
-            payload   = {}
-            url      = f"https://api.apilayer.com/image_to_text/url?url={image_url}"
-            response = requests.request("GET", url, headers=self.headers, data = payload)
+            payload = {}
+            url = f"https://api.apilayer.com/image_to_text/url?url={image_url}"
+            response = requests.request("GET", url, headers=self.headers, data=payload)
 
         status_code = response.status_code
-        rsp         = response.text
-        rsp       = ApiLayerResult(response.text, status_code)
-        metadata  = {}
+        rsp = response.text
+        rsp = ApiLayerResult(response.text, status_code)
+        metadata = {}
 
         return [rsp], metadata
 
     def prepare(self, argument):
         assert not argument.prop.processed_input, "OCREngine does not support processed_input."
-        image  = str(argument.prop.image)
+        image = str(argument.prop.image)
         argument.prop.prepared_input = image

symai/backend/engines/output/engine_stdout.py

@@ -7,10 +7,10 @@ class OutputEngine(Engine):
         self.name = self.__class__.__name__
 
     def id(self) -> str:
-        return 'output'
+        return "output"
 
     def forward(self, argument):
-        expr, processed, args, kwargs  = argument.prop.prepared_input
+        expr, processed, args, kwargs = argument.prop.prepared_input
         res = None
         args = [] if args is None else args
         kwargs = {} if kwargs is None else kwargs
@@ -18,14 +18,14 @@ class OutputEngine(Engine):
             res = expr(processed, *args, **kwargs) if processed else expr(*args, **kwargs)
 
         metadata = {}
-        result   = {
-            'result': res,
-            'processed': processed,
-            'args': args,
-            'kwargs': kwargs
-        }
+        result = {"result": res, "processed": processed, "args": args, "kwargs": kwargs}
 
         return [result], metadata
 
     def prepare(self, argument):
-        argument.prop.prepared_input = argument.prop.expr, argument.prop.processed_input, argument.prop.args, argument.prop.kwargs
+        argument.prop.prepared_input = (
+            argument.prop.expr,
+            argument.prop.processed_input,
+            argument.prop.args,
+            argument.prop.kwargs,
+        )

symai/backend/engines/{webscraping → scrape}/engine_requests.py

@@ -67,8 +67,8 @@ class RequestsEngine(Engine):
 
     DEFAULT_HEADERS: ClassVar[dict[str, str]] = {
         "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
-                      "AppleWebKit/537.36 (KHTML, like Gecko) "
-                      "Chrome/120.0.0.0 Safari/537.36",
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/120.0.0.0 Safari/537.36",
         "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
         "Accept-Language": "en-US,en;q=0.9",
         "DNT": "1",
@@ -174,7 +174,9 @@ class RequestsEngine(Engine):
             context.add_cookies(cookie_payload)
 
     @staticmethod
-    def _navigate_playwright_page(page, url: str, wait_selector: str | None, wait_until: str, timeout_ms: int, timeout_error):
+    def _navigate_playwright_page(
+        page, url: str, wait_selector: str | None, wait_until: str, timeout_ms: int, timeout_error
+    ):
         try:
             response = page.goto(url, wait_until=wait_until, timeout=timeout_ms)
             if wait_selector:
@@ -232,7 +234,13 @@ class RequestsEngine(Engine):
             return resp
         return self.session.get(target, timeout=timeout, allow_redirects=True)
 
-    def _fetch_with_playwright(self, url: str, wait_selector: str | None = None, wait_until: str = "networkidle", timeout: float | None = None):
+    def _fetch_with_playwright(
+        self,
+        url: str,
+        wait_selector: str | None = None,
+        wait_until: str = "networkidle",
+        timeout: float | None = None,
+    ):
         """
         Render the target URL in a headless browser to execute JavaScript and
         return a synthetic ``requests.Response`` object to keep downstream
@@ -240,11 +248,12 @@ class RequestsEngine(Engine):
         """
         try:
             # Playwright is optional; import only when JS rendering is requested.
-            from playwright.sync_api import TimeoutError as PlaywrightTimeoutError # noqa
-            from playwright.sync_api import sync_playwright # noqa
+            from playwright.sync_api import TimeoutError as PlaywrightTimeoutError  # noqa
+            from playwright.sync_api import sync_playwright  # noqa
+
             logging.getLogger("playwright").setLevel(logging.WARNING)
         except ImportError as exc:
-            msg = "Playwright is not installed. Install symbolicai[webscraping] with Playwright extras to enable render_js."
+            msg = "Playwright is not installed. Install symbolicai[scrape] with Playwright extras to enable render_js."
             UserMessage(msg)
             raise RuntimeError(msg) from exc
 
@@ -301,7 +310,7 @@ class RequestsEngine(Engine):
         return rendered_response
 
     def id(self) -> str:
-        return 'webscraping'
+        return "scrape"
 
     def forward(self, argument):
         """
@@ -317,8 +326,11 @@ class RequestsEngine(Engine):
         self._maybe_set_bypass_cookies(url)
 
         parsed = urlparse(url)
-        qs = [(k, v) for k, v in parse_qsl(parsed.query, keep_blank_values=True)
-              if k.lower() not in {"utm_source", "utm_medium", "utm_campaign"}]
+        qs = [
+            (k, v)
+            for k, v in parse_qsl(parsed.query, keep_blank_values=True)
+            if k.lower() not in {"utm_source", "utm_medium", "utm_campaign"}
+        ]
         clean_url = urlunparse(parsed._replace(query=urlencode(qs)))
 
         render_js = kwargs.get("render_js")
@@ -335,7 +347,9 @@ class RequestsEngine(Engine):
                 timeout=render_timeout,
             )
         else:
-            resp = self.session.get(clean_url, timeout=self.timeout, allow_redirects=True, verify=self.verify_ssl)
+            resp = self.session.get(
+                clean_url, timeout=self.timeout, allow_redirects=True, verify=self.verify_ssl
+            )
         resp.raise_for_status()
 
         # Follow a legacy meta refresh once (do AFTER normal HTTP redirects)

symai/backend/engines/search/engine_openai.py

@@ -21,9 +21,10 @@ logging.getLogger("httpcore").setLevel(logging.ERROR)
 
 
 TRACKING_KEYS = {
-    "utm_source" # so far I've only seen this one
+    "utm_source"  # so far I've only seen this one
 }
 
+
 @dataclass
 class Citation:
     id: int
@@ -33,14 +34,14 @@ class Citation:
     end: int
 
     def __hash__(self):
-        return hash((self.url, ))
+        return hash((self.url,))
 
 
 class SearchResult(Result):
     def __init__(self, value, **kwargs) -> None:
         super().__init__(value, **kwargs)
-        if value.get('error'):
-            UserMessage(value['error'], raise_with=ValueError)
+        if value.get("error"):
+            UserMessage(value["error"], raise_with=ValueError)
         try:
             text, annotations = self._extract_text_and_annotations(value)
             if text is None:
@@ -50,7 +51,9 @@ class SearchResult(Result):
             replaced_text, ordered, starts_ends = self._insert_citation_markers(text, annotations)
             self._value = replaced_text
             self._citations = [
-                Citation(id=cid, title=title, url=url, start=starts_ends[cid][0], end=starts_ends[cid][1])
+                Citation(
+                    id=cid, title=title, url=url, start=starts_ends[cid][0], end=starts_ends[cid][1]
+                )
                 for cid, title, url in ordered
             ]
 
@@ -59,70 +62,72 @@ class SearchResult(Result):
             UserMessage(f"Failed to parse response: {e}", raise_with=ValueError)
 
     def _extract_text(self, value) -> str | None:
-        if isinstance(value.get('output_text'), str) and value.get('output_text'):
-            return value.get('output_text')
+        if isinstance(value.get("output_text"), str) and value.get("output_text"):
+            return value.get("output_text")
         text = None
-        for output in value.get('output', []):
-            if output.get('type') == 'message' and output.get('content'):
-                content0 = output['content'][0]
-                if content0.get('text'):
-                    text = content0['text']
+        for output in value.get("output", []):
+            if output.get("type") == "message" and output.get("content"):
+                content0 = output["content"][0]
+                if content0.get("text"):
+                    text = content0["text"]
         return text
 
     def _extract_text_and_annotations(self, value):
         segments = []
         global_annotations = []
         pos = 0
-        for output in value.get('output', []) or []:
-            if output.get('type') != 'message' or not output.get('content'):
+        for output in value.get("output", []) or []:
+            if output.get("type") != "message" or not output.get("content"):
                 continue
-            for content in output.get('content', []) or []:
-                seg_text = content.get('text') or ''
+            for content in output.get("content", []) or []:
+                seg_text = content.get("text") or ""
                 if not isinstance(seg_text, str):
                     continue
-                for ann in (content.get('annotations') or []):
-                    if ann.get('type') == 'url_citation' and ann.get('url'):
-                        start = ann.get('start_index', 0)
-                        end = ann.get('end_index', 0)
-                        global_annotations.append({
-                            'type': 'url_citation',
-                            'url': ann.get('url'),
-                            'title': (ann.get('title') or '').strip(),
-                            'start_index': pos + int(start),
-                            'end_index': pos + int(end),
-                        })
+                for ann in content.get("annotations") or []:
+                    if ann.get("type") == "url_citation" and ann.get("url"):
+                        start = ann.get("start_index", 0)
+                        end = ann.get("end_index", 0)
+                        global_annotations.append(
+                            {
+                                "type": "url_citation",
+                                "url": ann.get("url"),
+                                "title": (ann.get("title") or "").strip(),
+                                "start_index": pos + int(start),
+                                "end_index": pos + int(end),
+                            }
+                        )
                 segments.append(seg_text)
                 pos += len(seg_text)
 
-        built_text = ''.join(segments) if segments else None
+        built_text = "".join(segments) if segments else None
         # Prefer top-level output_text if present AND segments are empty (no way to compute indices)
-        if not built_text and isinstance(value.get('output_text'), str):
-            return value.get('output_text'), []
+        if not built_text and isinstance(value.get("output_text"), str):
+            return value.get("output_text"), []
         return built_text, global_annotations
 
     def _normalize_url(self, u: str) -> str:
         parts = urlsplit(u)
         scheme = parts.scheme.lower()
         netloc = parts.netloc.lower()
-        path = parts.path.rstrip('/') or '/'
+        path = parts.path.rstrip("/") or "/"
         q = []
         for k, v in parse_qsl(parts.query, keep_blank_values=True):
             kl = k.lower()
-            if kl in TRACKING_KEYS or kl.startswith('utm_'):
+            if kl in TRACKING_KEYS or kl.startswith("utm_"):
                 continue
             q.append((k, v))
         query = urlencode(q, doseq=True)
-        fragment = ''
+        fragment = ""
         return urlunsplit((scheme, netloc, path, query, fragment))
 
     def _make_title_map(self, annotations):
         m = {}
         for a in annotations or []:
-            url = a.get('url')
+            url = a.get("url")
             if not url:
                 continue
             nu = self._normalize_url(url)
-            title = (a.get('title') or '').strip()
+            title = (a.get("title") or "").strip()
             if nu not in m and title:
                 m[nu] = title
         return m
@@ -131,7 +136,7 @@ class SearchResult(Result):
         return urlsplit(u).netloc
 
     def _short_hash_id(self, nu: str, length=6) -> str:
-        return hashlib.sha1(nu.encode('utf-8')).hexdigest()[:length]
+        return hashlib.sha1(nu.encode("utf-8")).hexdigest()[:length]
 
     def _insert_citation_markers(self, text: str, annotations):
         title_map = self._make_title_map(annotations)
@@ -140,8 +145,10 @@ class SearchResult(Result):
         ordered: list[tuple[int, str, str]] = []  # (id, title, normalized_url)
         next_id = 1
 
-        url_anns = [a for a in annotations or [] if a.get('type') == 'url_citation' and a.get('url')]
-        url_anns.sort(key=lambda a: int(a.get('start_index', 0)))
+        url_anns = [
+            a for a in annotations or [] if a.get("type") == "url_citation" and a.get("url")
+        ]
+        url_anns.sort(key=lambda a: int(a.get("start_index", 0)))
 
         pieces: list[str] = []
         cursor = 0
@@ -158,11 +165,11 @@ class SearchResult(Result):
             return id_map[nu]
 
         for ann in url_anns:
-            start = int(ann.get('start_index', 0))
-            end = int(ann.get('end_index', 0))
+            start = int(ann.get("start_index", 0))
+            end = int(ann.get("end_index", 0))
             if end <= cursor:
                 continue  # skip overlapping or backwards spans
-            url = ann.get('url')
+            url = ann.get("url")
             nu = self._normalize_url(url)
             cid = _get_id(nu)
             title = title_map.get(nu) or self._hostname(nu)
@@ -189,7 +196,7 @@ class SearchResult(Result):
 
         tail_clean = self._strip_markdown_links(text[cursor:])
         pieces.append(tail_clean)
-        replaced = ''.join(pieces)
+        replaced = "".join(pieces)
 
         starts_ends = {cid: first_span.get(cid, (0, 0)) for cid, _, _ in ordered}
         return replaced, ordered, starts_ends
@@ -197,15 +204,15 @@ class SearchResult(Result):
     def _strip_markdown_links(self, text: str) -> str:
         # Remove ([text](http...)) including surrounding parentheses
         pattern_paren = re.compile(r"\(\s*\[[^\]]+\]\(https?://[^)]+\)\s*\)")
-        text = pattern_paren.sub('', text)
+        text = pattern_paren.sub("", text)
         # Remove bare [text](http...)
         pattern_bare = re.compile(r"\[[^\]]+\]\(https?://[^)]+\)")
-        text = pattern_bare.sub('', text)
+        text = pattern_bare.sub("", text)
         # Remove parentheses that became empty or contain only commas/whitespace like (, , )
         pattern_empty_paren = re.compile(r"\(\s*\)")
-        text = pattern_empty_paren.sub('', text)
+        text = pattern_empty_paren.sub("", text)
         pattern_commas_only = re.compile(r"\(\s*(,\s*)+\)")
-        text = pattern_commas_only.sub('', text)
+        text = pattern_commas_only.sub("", text)
         # Collapse potential double spaces resulting from removals
         return re.sub(r"\s{2,}", " ", text).strip()
 
@@ -236,10 +243,12 @@ class GPTXSearchEngine(Engine):
         super().__init__()
         self.config = deepcopy(SYMAI_CONFIG)
         if api_key is not None and model is not None:
-            self.config['SEARCH_ENGINE_API_KEY'] = api_key
-            self.config['SEARCH_ENGINE_MODEL']   = model
-        self.api_key = self.config.get('SEARCH_ENGINE_API_KEY')
-        self.model = self.config.get('SEARCH_ENGINE_MODEL', 'gpt-4.1') # Default to gpt-4.1 as per docs
+            self.config["SEARCH_ENGINE_API_KEY"] = api_key
+            self.config["SEARCH_ENGINE_MODEL"] = model
+        self.api_key = self.config.get("SEARCH_ENGINE_API_KEY")
+        self.model = self.config.get(
+            "SEARCH_ENGINE_MODEL", "gpt-4.1"
+        )  # Default to gpt-4.1 as per docs
         self.name = self.__class__.__name__
         try:
             self.client = OpenAI(api_key=self.api_key)
@@ -247,9 +256,12 @@ class GPTXSearchEngine(Engine):
             UserMessage(f"Failed to initialize OpenAI client: {e}", raise_with=ValueError)
 
     def id(self) -> str:
-        if self.config.get('SEARCH_ENGINE_API_KEY') and \
-            self.config.get('SEARCH_ENGINE_MODEL') in OPENAI_CHAT_MODELS + OPENAI_REASONING_MODELS:
-            return 'search'
+        if (
+            self.config.get("SEARCH_ENGINE_API_KEY")
+            and self.config.get("SEARCH_ENGINE_MODEL")
+            in OPENAI_CHAT_MODELS + OPENAI_REASONING_MODELS
+        ):
+            return "search"
         return super().id()  # default to unregistered
 
     def _extract_netloc(self, raw_domain: str | None) -> str | None:
@@ -258,15 +270,15 @@ class GPTXSearchEngine(Engine):
         candidate = raw_domain.strip()
         if not candidate:
             return None
-        parsed = urlsplit(candidate if '://' in candidate else f"//{candidate}")
+        parsed = urlsplit(candidate if "://" in candidate else f"//{candidate}")
         netloc = parsed.netloc or parsed.path
         if not netloc:
             return None
-        if '@' in netloc:
-            netloc = netloc.split('@', 1)[1]
-        if ':' in netloc:
-            netloc = netloc.split(':', 1)[0]
-        netloc = netloc.strip('.').strip()
+        if "@" in netloc:
+            netloc = netloc.split("@", 1)[1]
+        if ":" in netloc:
+            netloc = netloc.split(":", 1)[0]
+        netloc = netloc.strip(".").strip()
         if not netloc:
             return None
         return netloc.lower()
@@ -313,38 +325,40 @@ class GPTXSearchEngine(Engine):
 
     def command(self, *args, **kwargs):
         super().command(*args, **kwargs)
-        if 'SEARCH_ENGINE_API_KEY' in kwargs:
-            self.api_key = kwargs['SEARCH_ENGINE_API_KEY']
-        if 'SEARCH_ENGINE_MODEL' in kwargs:
-            self.model = kwargs['SEARCH_ENGINE_MODEL']
+        if "SEARCH_ENGINE_API_KEY" in kwargs:
+            self.api_key = kwargs["SEARCH_ENGINE_API_KEY"]
+        if "SEARCH_ENGINE_MODEL" in kwargs:
+            self.model = kwargs["SEARCH_ENGINE_MODEL"]
 
     def forward(self, argument):
         messages = argument.prop.prepared_input
         kwargs = argument.kwargs
 
         tool_definition = {"type": "web_search"}
-        user_location = kwargs.get('user_location')
+        user_location = kwargs.get("user_location")
         if user_location:
-            tool_definition['user_location'] = user_location
+            tool_definition["user_location"] = user_location
 
-        allowed_domains = self._normalize_allowed_domains(kwargs.get('allowed_domains'))
+        allowed_domains = self._normalize_allowed_domains(kwargs.get("allowed_domains"))
         if allowed_domains:
-            tool_definition['filters'] = {
-                'allowed_domains': allowed_domains
-            }
+            tool_definition["filters"] = {"allowed_domains": allowed_domains}
 
-        self.model = kwargs.get('model', self.model) # Important for MetadataTracker to work correctly
+        self.model = kwargs.get(
+            "model", self.model
+        )  # Important for MetadataTracker to work correctly
 
         payload = {
             "model": self.model,
             "input": messages,
             "tools": [tool_definition],
-            "tool_choice": {"type": "web_search"} if self.model not in OPENAI_REASONING_MODELS else "auto" # force the use of web search tool for non-reasoning models
+            "tool_choice": {"type": "web_search"}
+            if self.model not in OPENAI_REASONING_MODELS
+            else "auto",  # force the use of web search tool for non-reasoning models
         }
 
         if self.model in OPENAI_REASONING_MODELS:
-            reasoning = kwargs.get('reasoning', { "effort": "low", "summary": "auto" })
-            payload['reasoning'] = reasoning
+            reasoning = kwargs.get("reasoning", {"effort": "low", "summary": "auto"})
+            payload["reasoning"] = reasoning
 
         try:
             res = self.client.responses.create(**payload)
@@ -353,21 +367,19 @@ class GPTXSearchEngine(Engine):
             UserMessage(f"Failed to make request: {e}", raise_with=ValueError)
 
         metadata = {"raw_output": res.raw}
-        output   = [res]
+        output = [res]
 
         return output, metadata
 
     def prepare(self, argument):
-        system_message = "You are a helpful AI assistant. Be precise and informative." if argument.kwargs.get('system_message') is None else argument.kwargs.get('system_message')
+        system_message = (
+            "You are a helpful AI assistant. Be precise and informative."
+            if argument.kwargs.get("system_message") is None
+            else argument.kwargs.get("system_message")
+        )
 
         res = [
-            {
-                "role": "system",
-                "content": system_message
-            },
-            {
-                "role": "user",
-                "content": f"{argument.prop.query}"
-            }
+            {"role": "system", "content": system_message},
+            {"role": "user", "content": f"{argument.prop.query}"},
         ]
         argument.prop.prepared_input = res