dhisana 0.0.1.dev243__py3-none-any.whl

dhisana/utils/serperdev_google_jobs.py

@@ -0,0 +1,125 @@
+import json
+import logging
+import os
+from typing import Any, Dict, List, Optional
+
+import aiohttp
+
+from dhisana.utils.assistant_tool_tag import assistant_tool
+from dhisana.utils.cache_output_tools import cache_output, retrieve_output
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def get_serper_dev_api_access_token(tool_config: Optional[List[Dict]] = None) -> str:
+    """Retrieve the SERPER_API_KEY from tool_config or environment.
+
+    Raises:
+        ValueError: If the Serper.dev integration has not been configured.
+    """
+    key = None
+    if tool_config:
+        cfg = next((c for c in tool_config if c.get("name") == "serperdev"), None)
+        if cfg:
+            kv = {i["name"]: i["value"] for i in cfg.get("configuration", [])}
+            key = kv.get("apiKey")
+    key = key or os.getenv("SERPER_API_KEY")
+    if not key:
+        raise ValueError(
+            "Serper.dev integration is not configured. Please configure the connection to Serper.dev in Integrations."
+        )
+    return key
+
+
+def _normalise_job_result(raw: Dict[str, Any]) -> Dict[str, Any]:
+    """Map a Serper job result onto a simplified schema."""
+    apply_link = ""
+    if isinstance(raw.get("apply_link"), str):
+        apply_link = raw.get("apply_link")
+    apply_options = raw.get("apply_options") or raw.get("apply_links") or []
+    if not apply_link and isinstance(apply_options, list) and apply_options:
+        first = apply_options[0]
+        if isinstance(first, dict):
+            apply_link = first.get("link") or first.get("apply_link") or ""
+
+    return {
+        "job_title": raw.get("title", ""),
+        "company_name": raw.get("company_name") or raw.get("company", ""),
+        "location": raw.get("location", ""),
+        "via": raw.get("via", ""),
+        "description": raw.get("description", ""),
+        "job_posting_url": raw.get("link") or apply_link,
+    }
+
+
+@assistant_tool
+async def search_google_jobs_serper(
+    query: str,
+    number_of_results: int = 10,
+    offset: int = 0,
+    tool_config: Optional[List[Dict]] = None,
+    location: Optional[str] = None,
+) -> List[str]:
+    """Search Google Jobs via Serper.dev and return normalised JSON strings."""
+    if not query:
+        logger.warning("Empty query provided to search_google_jobs_serper")
+        return []
+
+    cache_key = f"jobs_serper_{query}_{number_of_results}_{offset}_{location or ''}"
+    cached = retrieve_output("search_google_jobs_serper", cache_key)
+    if cached is not None:
+        return cached
+
+    api_key = get_serper_dev_api_access_token(tool_config)
+    headers = {"X-API-KEY": api_key, "Content-Type": "application/json"}
+    url = "https://google.serper.dev/search"  # ← fixed endpoint
+
+    page = offset + 1
+    collected: List[Dict[str, Any]] = []
+
+    async with aiohttp.ClientSession() as session:
+        while len(collected) < number_of_results:
+            payload = {
+                "q": query,
+                "page": page,
+                "type": "jobs",        # keeps us in the Jobs vertical
+                "autocorrect": True,
+                "gl": "us",
+                "hl": "en",
+            }
+            if location:
+                payload["location"] = location
+            try:
+                async with session.post(url, headers=headers, json=payload) as resp:
+                    if resp.status != 200:
+                        try:
+                            err = await resp.json()
+                        except Exception:
+                            err = await resp.text()
+                        logger.warning("Serper jobs error: %s", err)
+                        return [json.dumps({"error": err})]
+                    data = await resp.json()
+            except Exception as exc:
+                logger.exception("Serper jobs request failed")
+                return [json.dumps({"error": str(exc)})]
+
+            jobs = (
+                data.get("jobs")
+                or data.get("job_results")
+                or data.get("jobs_results")
+                or []
+            )
+            if not jobs:
+                break
+            collected.extend(jobs)
+            if len(collected) >= number_of_results:
+                break
+            page += 1
+
+    serialised = [
+        json.dumps(_normalise_job_result(j)) for j in collected[:number_of_results]
+    ]
+    cache_output("search_google_jobs_serper", cache_key, serialised)
+    logger.info("Returned %d job results for '%s'", len(serialised), query)
+    return serialised

dhisana/utils/serperdev_local_business.py

@@ -0,0 +1,154 @@
+import json
+import logging
+import os
+from typing import Any, Dict, List, Optional
+
+import aiohttp
+
+# Dhisana utils (adjust imports to your project structure)
+from dhisana.utils.assistant_tool_tag import assistant_tool
+from dhisana.utils.cache_output_tools import cache_output, retrieve_output
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
+
+
+# ────────────────────────────────────────────────────────────────
+# 1. API-key retrieval helper (mirrors your existing pattern)
+# ────────────────────────────────────────────────────────────────
+def get_serper_dev_api_access_token(tool_config: Optional[List[Dict]] = None) -> str:
+    """
+    Grab SERPER_API_KEY from tool_config or environment.
+
+    Raises:
+        ValueError: If the Serper.dev integration has not been configured.
+    """
+    SERPER_API_KEY = None
+    if tool_config:
+        serper_cfg = next(
+            (c for c in tool_config if c.get("name") == "serperdev"), None
+        )
+        if serper_cfg:
+            kv = {i["name"]: i["value"] for i in serper_cfg.get("configuration", [])}
+            SERPER_API_KEY = kv.get("apiKey")
+    SERPER_API_KEY = SERPER_API_KEY or os.getenv("SERPER_API_KEY")
+    if not SERPER_API_KEY:
+        raise ValueError(
+            "Serper.dev integration is not configured. Please configure the connection to Serper.dev in Integrations."
+        )
+    return SERPER_API_KEY
+
+
+# ────────────────────────────────────────────────────────────────
+# 2. Result normaliser
+# ────────────────────────────────────────────────────────────────
+def _normalise_local_result_serper(raw: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Map a Serper 'places' / 'placeResults' item onto Dhisana's schema.
+    """
+    cid = raw.get("cid") or raw.get("placeId") or raw.get("place_id")
+    maps_url = f"https://maps.google.com/?cid={cid}" if cid else ""
+
+    return {
+        "full_name":            raw.get("title", ""),
+        "organization_name":    raw.get("title", ""),
+        "phone":                raw.get("phoneNumber") or raw.get("phone", ""),
+        "organization_website": raw.get("website", ""),
+        "rating":               raw.get("rating"),
+        "reviews":              raw.get("reviews"),
+        "address":              raw.get("address", ""),
+        "google_maps_url":      maps_url,
+    }
+
+
+# ────────────────────────────────────────────────────────────────
+# 3. Search helper (decorated for Dhisana agents)
+# ────────────────────────────────────────────────────────────────
+@assistant_tool
+async def search_local_business_serper(
+    query: str,
+    number_of_results: int = 20,
+    offset: int = 0,
+    tool_config: Optional[List[Dict]] = None,
+    location: Optional[str] = None,
+) -> List[str]:
+    """
+    Fetch Google-Maps local business results via Serper.dev and return a
+    List[str] of JSON-encoded business objects in Dhisana's schema.
+
+    Args:
+        query: Main search string (e.g. "coffee shops").
+        number_of_results: Total rows desired.
+        offset: Page offset (page index starts at 0).
+        tool_config: Optional Dhisana tool-config containing the API key.
+        location: Optional "San Jose, CA" style hint to refine results.
+    """
+    if not query:
+        logger.warning("Empty query.")
+        return []
+
+    # ── caching
+    cache_key = f"local_serper_{query}_{number_of_results}_{offset}_{location or ''}"
+    cached = retrieve_output("search_local_serper", cache_key)
+    if cached is not None:
+        return cached
+
+    api_key = get_serper_dev_api_access_token(tool_config)
+    headers = {"X-API-KEY": api_key, "Content-Type": "application/json"}
+    url = "https://google.serper.dev/places"
+
+    page_size = 20  # Serper returns ≤20 rows per page
+    page = offset + 1
+    collected: List[Dict[str, Any]] = []
+
+    async with aiohttp.ClientSession() as session:
+        while len(collected) < number_of_results:
+            payload = {
+                "q": query,
+                "page": page,
+                "type": "places",   # explicit although /places path implies it
+                "autocorrect": True,
+                "gl": "us",
+                "hl": "en",
+            }
+            if location:
+                payload["location"] = location
+
+            try:
+                async with session.post(url, headers=headers, json=payload) as resp:
+                    if resp.status != 200:
+                        # Bubble up API errors
+                        try:
+                            err = await resp.json()
+                        except Exception:
+                            err = await resp.text()
+                        logger.warning("Serper Places error: %s", err)
+                        return [json.dumps({"error": err})]
+                    data = await resp.json()
+            except Exception as exc:
+                logger.exception("Serper Places request failed.")
+                return [json.dumps({"error": str(exc)})]
+
+            # Handle both field names
+            places = (
+                data.get("places")
+                or data.get("placeResults")
+                or data.get("local_results")
+                or []
+            )
+            if not places:
+                break
+
+            collected.extend(places)
+            if len(collected) >= number_of_results:
+                break
+            page += 1
+
+    # normalise → serialise
+    serialised = [
+        json.dumps(_normalise_local_result_serper(p))
+        for p in collected[:number_of_results]
+    ]
+    cache_output("search_local_serper", cache_key, serialised)
+    logger.info("Returned %d local businesses for '%s'", len(serialised), query)
+    return serialised

dhisana/utils/serperdev_search.py

@@ -0,0 +1,233 @@
+import asyncio
+import json
+import os
+from typing import Any, Dict, List, Optional
+import aiohttp
+
+import logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+# If you have these utils in your project, import them; otherwise, remove them or replace them.
+from dhisana.utils.cache_output_tools import cache_output, retrieve_output
+
+
+def get_serper_dev_api_access_token(tool_config: Optional[List[Dict]] = None) -> str:
+    """
+    Retrieves the Serper.dev API access token from the provided tool configuration
+    or from the SERPER_API_KEY environment variable.
+
+    Args:
+        tool_config (list): A list of dictionaries containing the tool configuration.
+                            Each dictionary should have a "name" key and a "configuration" key,
+                            where "configuration" is a list of dictionaries containing "name" and "value" keys.
+
+    Returns:
+        str: The Serper.dev API access token.
+
+    Raises:
+        ValueError: If the Serper.dev integration has not been configured.
+    """
+    logger.info("Entering get_serper_dev_api_access_token")
+    SERPER_API_KEY = None
+
+    if tool_config:
+        logger.debug(f"Tool config provided: {tool_config}")
+        serper_config = next(
+            (item for item in tool_config if item.get("name") == "serperdev"), None
+        )
+        if serper_config:
+            config_map = {
+                item["name"]: item["value"]
+                for item in serper_config.get("configuration", [])
+                if item
+            }
+            SERPER_API_KEY = config_map.get("apiKey")
+        else:
+            logger.warning("No 'serperdev' config item found in tool_config.")
+    else:
+        logger.debug("No tool_config provided or it's None.")
+
+    SERPER_API_KEY = SERPER_API_KEY or os.getenv("SERPER_API_KEY")
+    if not SERPER_API_KEY:
+        logger.error("Serper.dev integration is not configured.")
+        raise ValueError(
+            "Serper.dev integration is not configured. Please configure the connection to Serper.dev in Integrations."
+        )
+
+    logger.info("Retrieved SERPER_API_KEY successfully.")
+    return SERPER_API_KEY
+
+
+
+async def search_google_serper(
+    query: str,
+    number_of_results: int = 10,
+    offset: int = 0,
+    tool_config: Optional[List[Dict]] = None,
+    as_oq: Optional[str] = None
+) -> List[str]:
+    """
+    Search Google using Serper.dev. Mimics the signature and usage of the old SerpAPI function,
+    and normalizes the response JSON objects so that they contain:
+      - "title"
+      - "link"
+      - "snippet"
+      - "position"
+
+    This ensures consistency with SerpAPI-based code.
+    
+    Parameters:
+    - query (str): The search query.
+    - number_of_results (int): The total number of results to return. Default is 10.
+    - offset (int): The "page offset" to start from (used to compute the page).
+    - tool_config (Optional[List[Dict]]): Configuration containing the Serper.dev API token, etc.
+    - as_oq (Optional[str]): Optional additional query terms, appended to 'query'.
+    
+    Returns:
+    - List[str]: A list of organic search results, each serialized as a JSON string 
+                 with "title", "link", "snippet", and "position".
+    """
+    logger.info("Entering search_google_serper")
+
+    if not query:
+        logger.warning("Empty query string provided.")
+        return []
+
+    # Combine main query with optional terms
+    full_query = query
+    if as_oq:
+        full_query += f" {as_oq}"
+
+    # Check cache
+    cache_key = f"{full_query}_{number_of_results}_{offset}"
+    cached_response = retrieve_output("search_google_serper", cache_key)
+    if cached_response is not None:
+        logger.info("Cache hit for search_google_serper.")
+        return cached_response
+
+    # Retrieve your Serper.dev API key (replace with your own function if needed)
+    SERPER_API_KEY = get_serper_dev_api_access_token(tool_config)
+
+    url = "https://google.serper.dev/search"
+    headers = {
+        "X-API-KEY": SERPER_API_KEY,
+        "Content-Type": "application/json"
+    }
+
+    # Serper.dev uses 'page' to paginate. We need to calculate the page number based on offset.
+    # Serper.dev uses 1-based indexing for pages, so we need to adjust accordingly
+    # e.g., if offset is 0, we want page 1; if offset is 10, we want page 2, etc.
+    # Assuming number_of_results is the number of results per page.
+    # If offset is 0, we want the first page,
+    # if offset is 10 and number_of_results is 10, we want the second page, etc.
+    # This means we can calculate the page as follows:
+    # page = (offset // number_of_results) + 1
+    # If offset is 0, page will be 1; if offset is 10 and number_of_results is 10, page will be 2.
+    # This is consistent with how Serper.dev handles pagination.
+    page = 1 if offset == 0 else (offset // number_of_results) + 1
+    all_results: List[Dict[str, Any]] = []
+
+    # We'll collect results from "organic", converting each to a SerpAPI-like format.
+    timeout = aiohttp.ClientTimeout(total=30, connect=10)
+    async with aiohttp.ClientSession(timeout=timeout) as session:
+        while len(all_results) < number_of_results:
+            payload = {
+                "q": full_query,
+                "gl": "us",        # geolocation
+                "hl": "en",        # language
+                "autocorrect": True,
+                "page": page,
+                "num": number_of_results,
+                "type": "search"   # or 'news', 'images', etc., if needed
+            }
+
+            logger.debug(f"Requesting Serper.dev page {page} for query '{full_query}'.")
+            for attempt in range(3):
+                try:
+                    async with session.post(url, headers=headers, json=payload) as response:
+                        if response.status != 200:
+                            try:
+                                error_content = await response.json()
+                            except Exception:
+                                error_content = await response.text()
+                            logger.warning(
+                                "Non-200 response from Serper.dev: %s (status=%s)",
+                                error_content,
+                                response.status,
+                            )
+                            return [json.dumps({"error": error_content})]
+
+                        result_json = await response.json()
+                        break
+                except asyncio.TimeoutError:
+                    logger.warning(
+                        "Timeout contacting Serper.dev (attempt %s/3) for query '%s'",
+                        attempt + 1,
+                        full_query,
+                    )
+                except aiohttp.ClientError as exc:
+                    logger.warning(
+                        "Client error contacting Serper.dev (attempt %s/3): %s",
+                        attempt + 1,
+                        exc,
+                    )
+                    if attempt == 2:
+                        logger.exception("Exception during Serper.dev request.")
+                        return [json.dumps({"error": str(exc)})]
+                except Exception as e:
+                    logger.exception("Unexpected exception during Serper.dev request.")
+                    return [json.dumps({"error": str(e)})]
+                else:
+                    # Successful request, exit retry loop
+                    break
+                await asyncio.sleep(2 ** attempt)
+            else:
+                logger.error(
+                    "Failed to retrieve data from Serper.dev after multiple attempts for query '%s'",
+                    full_query,
+                )
+                return [
+                    json.dumps(
+                        {
+                            "error": "Serper.dev request timed out after multiple attempts.",
+                        }
+                    )
+                ]
+
+            organic_results = result_json.get("organic", [])
+            if not organic_results:
+                logger.debug("No more organic results returned; stopping.")
+                break
+
+            all_results.extend(organic_results)
+            page += 1
+
+            if len(all_results) >= number_of_results:
+                break
+
+    # Limit to the requested number_of_results
+    all_results = all_results[:number_of_results]
+
+    # Convert each Serper.dev result to a SerpAPI-like format
+    # SerpAPI typically returns objects with keys: "position", "title", "link", "snippet", etc.
+    normalized_results = []
+    for idx, item in enumerate(all_results):
+        # item from Serper.dev might have: { "title": "...", "link": "...", "snippet": "..." }
+        # If the field name for snippet is different, change accordingly. 
+        # But as of serper.dev docs, "snippet" is used.
+        normalized_item = {
+            "position": idx + 1,
+            "title": item.get("title", ""),
+            "link": item.get("link", ""),
+            "snippet": item.get("snippet", ""),
+            # Copy any other fields if you want them
+        }
+        normalized_results.append(json.dumps(normalized_item))
+
+    logger.info(f"Found {len(normalized_results)} normalized results for query '{full_query}'.")
+    cache_output("search_google_serper", cache_key, normalized_results)
+
+    return normalized_results
+