webscout 8.2.6__py3-none-any.whl → 8.2.8__py3-none-any.whl

webscout/Provider/AISEARCH/genspark_search.py

@@ -2,7 +2,8 @@ import cloudscraper
 from uuid import uuid4
 import json
 import re
-from typing import Dict, Optional, Generator, Union, Any
+from typing import TypedDict, List, Iterator, cast, Dict, Optional, Generator, Union, Any
+import requests
 
 from webscout.AIbase import AISearch
 from webscout import exceptions
@@ -35,57 +36,124 @@ class Response:
         return self.text
 
 
+class SourceDict(TypedDict, total=False):
+    url: str
+    title: str
+    snippet: str
+    favicon: str
+    # Add more fields as needed
+
+class StatusUpdateDict(TypedDict):
+    type: str
+    message: str
+
+class StatusTopBarDict(TypedDict, total=False):
+    type: str
+    data: dict
+
+class PeopleAlsoAskDict(TypedDict, total=False):
+    question: str
+    answer: str
+
+class ResultSummaryDict(TypedDict, total=False):
+    source: str
+    rel_score: float
+    score: float
+    llm_id: str
+    cogen_name: str
+    ended: bool
+
 class Genspark(AISearch):
-    """A class to interact with the Genspark AI search API.
+    """
+    Strongly typed Genspark AI search API client.
     
     Genspark provides a powerful search interface that returns AI-generated responses
     based on web content. It supports both streaming and non-streaming responses.
+
+    After a search, several attributes are populated with extracted data:
+    - `search_query_details` (dict): Information about the classified search query.
+    - `status_updates` (list): Log of status messages during the search.
+    - `final_search_results` (list): Organic search results if provided by the API.
+    - `sources_used` (list): Unique web sources used for the answer.
+    - `people_also_ask` (list): "People Also Ask" questions.
+    - `agents_guide` (dict): Information about agents used.
+    - `result_summary` (dict): Summary of result IDs and scores.
+    - `raw_events_log` (list): If enabled, logs all raw JSON events from the stream.
     
     Basic Usage:
         >>> from webscout import Genspark
         >>> ai = Genspark()
-        >>> # Non-streaming example
-        >>> response = ai.search("What is Python?")
-        >>> print(response)
+        >>> # Non-streaming example (text response)
+        >>> response_text = ai.search("What is Python?")
+        >>> print(response_text)
         Python is a high-level programming language...
+        >>> # Access additional data:
+        >>> # print(ai.sources_used)
         
-        >>> # Streaming example
-        >>> for chunk in ai.search("Tell me about AI", stream=True):
-        ...     print(chunk, end="", flush=True)
+        >>> # Streaming example (mixed content: text Response objects and event dicts)
+        >>> for item in ai.search("Tell me about AI", stream=True):
+        ...     if isinstance(item, Response):
+        ...         print(item, end="", flush=True)
+        ...     else:
+        ...         print(f"\n[EVENT: {item.get('event')}]") 
         Artificial Intelligence is...
+        [EVENT: status_update]
+        ...
         
-        >>> # Raw response format
-        >>> for chunk in ai.search("Hello", stream=True, raw=True):
-        ...     print(chunk)
-        {'text': 'Hello'}
-        {'text': ' there!'}
+        >>> # Raw streaming response format
+        >>> for raw_event_dict in ai.search("Hello", stream=True, raw=True):
+        ...     print(raw_event_dict)
+        {'type': 'result_start', ...}
+        {'type': 'result_field_delta', 'field_name': 'streaming_detail_answer[0]', 'delta': 'Hello', ...}
     
     Args:
         timeout (int, optional): Request timeout in seconds. Defaults to 30.
         proxies (dict, optional): Proxy configuration for requests. Defaults to None.
-        max_tokens (int, optional): Maximum tokens to generate. Defaults to 600.
+        max_tokens (int, optional): Maximum tokens to generate (Note: This param is part of Genspark class but not directly used in API call shown). Defaults to 600.
+        log_raw_events (bool, optional): If True, all raw JSON events from the stream are logged to `self.raw_events_log`. Defaults to False.
     """
 
+    session: cloudscraper.CloudScraper
+    max_tokens: int
+    chat_endpoint: str
+    stream_chunk_size: int
+    timeout: int
+    log_raw_events: bool
+    headers: Dict[str, str]
+    cookies: Dict[str, str]
+    last_response: Union[Response, Dict[str, Any], List[Any], None]
+    search_query_details: Dict[str, Any]
+    status_updates: List[StatusUpdateDict]
+    final_search_results: Optional[List[Any]]
+    sources_used: List[SourceDict]
+    _seen_source_urls: set
+    people_also_ask: List[PeopleAlsoAskDict]
+    _seen_paa_questions: set
+    agents_guide: Optional[List[Any]]
+    result_summary: Dict[str, ResultSummaryDict]
+    raw_events_log: List[dict]
+
     def __init__(
         self,
         timeout: int = 30,
-        proxies: Optional[dict] = None,
+        proxies: Optional[Dict[str, str]] = None,
         max_tokens: int = 600,
-    ):
+        log_raw_events: bool = False,
+    ) -> None:
         """Initialize the Genspark API client.
         
         Args:
             timeout (int, optional): Request timeout in seconds. Defaults to 30.
             proxies (dict, optional): Proxy configuration for requests. Defaults to None.
             max_tokens (int, optional): Maximum tokens to generate. Defaults to 600.
+            log_raw_events (bool, optional): Log all raw events to self.raw_events_log. Defaults to False.
         """
         self.session = cloudscraper.create_scraper()
         self.max_tokens = max_tokens
         self.chat_endpoint = "https://www.genspark.ai/api/search/stream"
         self.stream_chunk_size = 64
         self.timeout = timeout
-        self.last_response = {}
-        
+        self.log_raw_events = log_raw_events
         self.headers = {
             "Accept": "*/*",
             "Accept-Encoding": "gzip, deflate, br, zstd",
@@ -102,49 +170,61 @@ class Genspark(AISearch):
             "Sec-Fetch-Site": "same-origin",
             "User-Agent": LitAgent().random(),
         }
-        
         self.cookies = {
             "i18n_redirected": "en-US",
-            "agree_terms": "0",
+            "agree_terms": "0", # Note: Ensure this cookie reflects actual consent if needed
             "session_id": uuid4().hex,
         }
-
         self.session.headers.update(self.headers)
         self.session.proxies = proxies or {}
+        self.last_response = None
+        self._reset_search_data()
+
+    def _reset_search_data(self) -> None:
+        """Resets attributes that store data from a search stream."""
+        self.search_query_details = {}
+        self.status_updates = []
+        self.final_search_results = None
+        self.sources_used = []
+        self._seen_source_urls = set()
+        self.people_also_ask = []
+        self._seen_paa_questions = set()
+        self.agents_guide = None
+        self.result_summary = {}
+        self.raw_events_log = []
 
     def search(
         self,
         prompt: str,
         stream: bool = False,
         raw: bool = False,
-    ) -> Union[Response, Generator[Union[Dict[str, str], Response], None, None]]:
-        """Search using the Genspark API and get AI-generated responses.
-        
+    ) -> Union[
+        Response,
+        Dict[str, Any],
+        List[dict],
+        Iterator[Union[dict, Response]],
+    ]:
+        """
+        Strongly typed search method for Genspark API.
         Args:
-            prompt (str): The search query or prompt to send to the API.
-            stream (bool, optional): If True, yields response chunks as they arrive.
-                                   If False, returns complete response. Defaults to False.
-            raw (bool, optional): If True, returns raw response dictionaries with 'text' key.
-                                If False, returns Response objects that convert to text automatically.
-                                Defaults to False.
-        
+            prompt: The search query or prompt.
+            stream: If True, yields results as they arrive.
+            raw: If True, yields/returns raw event dicts.
         Returns:
-            Union[Dict[str, Any], Generator[Union[Dict[str, Any], str], None, None]]: 
-                - If stream=False: Returns complete response
-                - If stream=True: Yields response chunks as they arrive
-        
-        Raises:
-            APIConnectionError: If the API request fails
+            - If stream=True, raw=True: Iterator[dict]
+            - If stream=True, raw=False: Iterator[Response | dict]
+            - If stream=False, raw=True: List[dict]
+            - If stream=False, raw=False: Response
         """
-        url = f"https://www.genspark.ai/api/search/stream?query={prompt.replace(' ', '%20')}"
-        
-        def for_stream():
+        self._reset_search_data()
+        url = f"{self.chat_endpoint}?query={requests.utils.quote(prompt)}"
+        def _process_stream() -> Iterator[Union[dict, Response]]:
             try:
                 with self.session.post(
                     url,
                     headers=self.headers,
                     cookies=self.cookies,
-                    json={},  # Empty payload as query is in URL
+                    json={},
                     stream=True,
                     timeout=self.timeout,
                 ) as response:
@@ -152,57 +232,119 @@ class Genspark(AISearch):
                         raise exceptions.APIConnectionError(
                             f"Failed to generate response - ({response.status_code}, {response.reason}) - {response.text}"
                         )
-                    
                     for line in response.iter_lines(decode_unicode=True):
-                        if line and line.startswith("data: "):
-                            try:
-                                data = json.loads(line[6:])
-                                if "field_name" in data and "delta" in data:
-                                    if data["field_name"].startswith("streaming_detail_answer"):
-                                        delta_text = data.get("delta", "")
-                                        
-                                        # Clean up markdown links in text
-                                        delta_text = re.sub(r"\[.*?\]\(.*?\)", "", delta_text)
-                                        
-                                        if raw:
-                                            yield {"text": delta_text}
-                                        else:
-                                            yield Response(delta_text)
-                            except json.JSONDecodeError:
-                                continue
-                                
+                        if not line or not line.startswith("data: "):
+                            continue
+                        try:
+                            data = json.loads(line[6:])
+                            if self.log_raw_events:
+                                self.raw_events_log.append(data)
+                            event_type = data.get("type")
+                            field_name = data.get("field_name")
+                            result_id = data.get("result_id")
+                            if raw:
+                                yield data
+                            # Populate instance attributes
+                            if event_type == "result_start":
+                                self.result_summary[result_id] = cast(ResultSummaryDict, {
+                                    "source": data.get("result_source"),
+                                    "rel_score": data.get("result_rel_score"),
+                                    "score": data.get("result_score"),
+                                    "llm_id": data.get("llm_id"),
+                                    "cogen_name": data.get("cogen", {}).get("name"),
+                                })
+                            elif event_type == "classify_query_result":
+                                self.search_query_details["classification"] = data.get("classify_query_result")
+                            elif event_type == "result_field":
+                                field_value = data.get("field_value")
+                                if field_name == "search_query":
+                                    self.search_query_details["query_string"] = field_value
+                                elif field_name == "thinking":
+                                    self.status_updates.append({"type": "thinking", "message": field_value})
+                                elif field_name == "search_status_top_bar_data":
+                                    self.status_updates.append({"type": "status_top_bar", "data": field_value})
+                                    if isinstance(field_value, dict) and field_value.get("status") == "finished":
+                                        self.final_search_results = field_value.get("search_results")
+                                        if field_value.get("search_plan"):
+                                            self.search_query_details["search_plan"] = field_value.get("search_plan")
+                                elif field_name == "search_source_top_bar_data":
+                                    if isinstance(field_value, list):
+                                        for source in field_value:
+                                            if isinstance(source, dict) and source.get("url") and source.get("url") not in self._seen_source_urls:
+                                                self.sources_used.append(cast(SourceDict, source))
+                                                self._seen_source_urls.add(source.get("url"))
+                            elif event_type == "result_end":
+                                if result_id in self.result_summary:
+                                    self.result_summary[result_id]["ended"] = True
+                                search_result_data = data.get("search_result")
+                                if search_result_data and isinstance(search_result_data, dict):
+                                    if search_result_data.get("source") == "people_also_ask" and "people_also_ask" in search_result_data:
+                                        paa_list = search_result_data["people_also_ask"]
+                                        if isinstance(paa_list, list):
+                                            for paa_item in paa_list:
+                                                if isinstance(paa_item, dict) and paa_item.get("question") not in self._seen_paa_questions:
+                                                    self.people_also_ask.append(cast(PeopleAlsoAskDict, paa_item))
+                                                    self._seen_paa_questions.add(paa_item.get("question"))
+                                    elif search_result_data.get("source") == "agents_guide" and "agents_guide" in search_result_data:
+                                        self.agents_guide = search_result_data["agents_guide"]
+                            if not raw:
+                                processed_event_payload = None
+                                if event_type == "result_field_delta" and field_name and field_name.startswith("streaming_detail_answer"):
+                                    delta_text = data.get("delta", "")
+                                    delta_text = re.sub(r"\[.*?\]\(.*?\)", "", delta_text)
+                                    yield Response(delta_text)
+                                elif event_type == "result_start":
+                                    processed_event_payload = {"event": "result_start", "data": {"id": result_id, "source": data.get("result_source"), "score": data.get("result_score")}}
+                                elif event_type == "classify_query_result":
+                                    processed_event_payload = {"event": "query_classification", "data": data.get("classify_query_result")}
+                                elif event_type == "result_field":
+                                    field_value = data.get("field_value")
+                                    if field_name == "search_query":
+                                        processed_event_payload = {"event": "search_query_update", "value": field_value}
+                                    elif field_name == "thinking":
+                                         processed_event_payload = {"event": "thinking_update", "value": field_value}
+                                    elif field_name == "search_status_top_bar_data":
+                                        processed_event_payload = {"event": "status_update", "data": field_value}
+                                    elif field_name == "search_source_top_bar_data":
+                                         processed_event_payload = {"event": "sources_update", "data": field_value}
+                                elif event_type == "result_end":
+                                    processed_event_payload = {"event": "result_end", "data": {"id": result_id, "search_result": data.get("search_result")}}
+                                if processed_event_payload:
+                                    yield processed_event_payload
+                        except json.JSONDecodeError:
+                            continue
             except cloudscraper.exceptions as e:
+                raise exceptions.APIConnectionError(f"Request failed due to Cloudscraper issue: {e}")
+            except requests.exceptions.RequestException as e:
                 raise exceptions.APIConnectionError(f"Request failed: {e}")
-                
-        def for_non_stream():
-            full_response = ""
-            for chunk in for_stream():
+        processed_stream_gen = _process_stream()
+        if stream:
+            return processed_stream_gen
+        else:
+            full_response_text = ""
+            all_raw_events_for_this_search: List[dict] = []
+            for item in processed_stream_gen:
                 if raw:
-                    yield chunk
+                    all_raw_events_for_this_search.append(cast(dict, item))
                 else:
-                    full_response += str(chunk)
-            
-            if not raw:
-                # Process the full response to clean up any JSON structures
-                try:
-                    text_json = json.loads(full_response)
-                    if isinstance(text_json, dict) and "detailAnswer" in text_json:
-                        full_response = text_json.get("detailAnswer", full_response)
-                except (json.JSONDecodeError, TypeError):
-                    # Not valid JSON or not a dictionary, keep as is
-                    pass
-                    
-                self.last_response = Response(full_response)
-                return self.last_response
-
-        return for_stream() if stream else for_non_stream()
-
+                    if isinstance(item, Response):
+                        full_response_text += str(item)
+            if raw:
+                self.last_response = {"raw_events": all_raw_events_for_this_search}
+                return all_raw_events_for_this_search
+            else:
+                final_text_response = Response(full_response_text)
+                self.last_response = final_text_response
+                return final_text_response
 
 if __name__ == "__main__":
-
     from rich import print
-    
     ai = Genspark()
-    response = ai.search(input(">>> "), stream=True, raw=False)
-    for chunk in response:
-        print(chunk, end="", flush=True)
+    try:
+        response = ai.search(input(">>> "), stream=True, raw=False)
+        for chunk in response:
+            print(chunk, end="", flush=True)
+    except KeyboardInterrupt:
+        print("\nSearch interrupted by user.")
+    except Exception as e:
+        print(f"\nError: {e}")

webscout/Provider/AISEARCH/hika_search.py

@@ -31,7 +31,7 @@ class Hika(AISearch):
         timeout: int = 60,
         proxies: Optional[dict] = None,
         language: str = "en",
-        model: str = "deepseek-r1",
+        # model: str = "deepseek-r1",
         
     ):
         self.session = requests.Session()
@@ -39,7 +39,7 @@ class Hika(AISearch):
         self.endpoint = "kbase/web"
         self.timeout = timeout
         self.language = language
-        self.model = model
+        # self.model = model
         self.last_response = {}
         
         self.headers = {
@@ -104,12 +104,12 @@ class Hika(AISearch):
             "x-uid": uid
         }
         
-        # Prepare payload
+        # Prepare payload (fix: stream as string, add search_language)
         payload = {
             "keyword": prompt,
-            "model": self.model,
             "language": self.language,
-            "stream": True  # Always request streaming for consistent handling
+            "search_language": self.language,
+            "stream": "true"  # Must be string, not boolean
         }
 
         def for_stream():
@@ -131,21 +131,21 @@ class Hika(AISearch):
                         if line and line.startswith("data: "):
                             try:
                                 data = json.loads(line[6:])
+                                # Handle chunk and references
                                 if "chunk" in data:
                                     chunk = data["chunk"]
-                                    
-                                    # Skip [DONE] markers completely
                                     if "[DONE]" in chunk:
                                         continue
-                                    
-                                    # Clean the chunk
                                     clean_chunk = self.clean_text(chunk)
-                                    
-                                    if clean_chunk:  # Only yield if there's content after cleaning
+                                    if clean_chunk:
                                         if raw:
                                             yield {"text": clean_chunk}
                                         else:
                                             yield Response(clean_chunk)
+                                elif "references" in data:
+                                    # Optionally yield references if raw requested
+                                    if raw:
+                                        yield {"references": data["references"]}
                             except json.JSONDecodeError:
                                 pass