dhisana 0.0.1.dev85__py3-none-any.whl → 0.0.1.dev236__py3-none-any.whl

dhisana/utils/serpapi_google_jobs.py

@@ -0,0 +1,117 @@
+import json
+import logging
+from typing import Any, Dict, List, Optional
+
+import aiohttp
+
+from dhisana.utils.assistant_tool_tag import assistant_tool
+from dhisana.utils.cache_output_tools import cache_output, retrieve_output
+from dhisana.utils.serpapi_google_search import get_serp_api_access_token
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def _normalise_job_result(raw: Dict[str, Any]) -> Dict[str, Any]:
+    """Convert a SerpApi jobs result to a simplified schema."""
+    # ...existing code...
+    apply_link = ""
+    apply_options = raw.get("apply_options") or raw.get("apply_links") or []
+    if isinstance(apply_options, list) and apply_options:
+        first = apply_options[0]
+        if isinstance(first, dict):
+            apply_link = first.get("link") or first.get("apply_link") or ""
+    if isinstance(apply_options, dict):
+        apply_link = apply_options.get("link") or apply_options.get("apply_link") or ""
+
+    return {
+        "job_title": raw.get("title", ""),
+        "organization_name": raw.get("company_name") or raw.get("company", ""),
+        "location": raw.get("location", ""),
+        "via": raw.get("via", ""),
+        "description": raw.get("description", ""),
+        "job_posting_url": raw.get("job_highlight_url")
+        or raw.get("apply_link")
+        or apply_link
+        or raw.get("link", ""),
+    }
+
+
+@assistant_tool
+async def search_google_jobs_serpapi(
+    query: str,
+    number_of_results: int = 10,
+    offset: int = 0,
+    tool_config: Optional[List[Dict]] = None,
+    location: Optional[str] = None,
+) -> List[str]:
+    """Search Google Jobs via SerpApi and return normalised JSON strings."""
+    if not query:
+        logger.warning("Empty query provided to search_google_jobs_serpapi")
+        return []
+
+    cache_key = f"jobs_serpapi_{query}_{number_of_results}_{offset}_{location or ''}"
+    cached = retrieve_output("search_google_jobs_serpapi", cache_key)
+    if cached is not None:
+        return cached
+
+    api_key = get_serp_api_access_token(tool_config)
+    page_size = 10
+    collected: List[Dict[str, Any]] = []
+    next_page_token = None
+
+    async with aiohttp.ClientSession() as session:
+        while len(collected) < number_of_results:
+            to_fetch = min(page_size, number_of_results - len(collected))
+            params = {
+                "engine": "google_jobs",
+                "q": query,
+                "api_key": api_key,
+                "num": to_fetch,
+            }
+            
+            if offset > 0 and next_page_token is None:
+                # If offset is provided but we don't have a token yet,
+                # we can't properly paginate. Log a warning.
+                logger.warning("Offset provided but next_page_token not available. Results may be incomplete.")
+                
+            if next_page_token:
+                params["next_page_token"] = next_page_token
+                
+            if location:
+                params["location"] = location
+                
+            try:
+                async with session.get("https://serpapi.com/search", params=params) as resp:
+                    if resp.status != 200:
+                        try:
+                            err = await resp.json()
+                        except Exception:
+                            err = await resp.text()
+                        logger.warning("SerpApi jobs error: %s", err)
+                        return [json.dumps({"error": err})]
+                    payload = await resp.json()
+            except Exception as exc:
+                logger.exception("SerpApi jobs request failed")
+                return [json.dumps({"error": str(exc)})]
+
+            jobs = payload.get("jobs_results", [])
+            if not jobs:
+                break
+                
+            collected.extend(jobs)
+            
+            # Get the next_page_token for pagination
+            next_page_token = payload.get("serpapi_pagination", {}).get("next_page_token")
+            if not next_page_token:
+                break
+            
+            # If we've collected enough results, stop pagination
+            if len(collected) >= number_of_results:
+                break
+
+    normalised = [_normalise_job_result(j) for j in collected[:number_of_results]]
+    serialised = [json.dumps(item) for item in normalised]
+    cache_output("search_google_jobs_serpapi", cache_key, serialised)
+    logger.info("Returned %d job results for '%s'", len(serialised), query)
+    return serialised

dhisana/utils/serpapi_google_search.py

@@ -0,0 +1,188 @@
+import json
+import os
+from typing import Dict, List, Optional
+import aiohttp
+
+from dhisana.utils.assistant_tool_tag import assistant_tool
+from dhisana.utils.cache_output_tools import cache_output, retrieve_output
+
+import logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def get_serp_api_access_token(tool_config: Optional[List[Dict]] = None) -> str:
+    """
+    Retrieves the SERPAPI_KEY access token from the provided tool configuration.
+
+    Args:
+        tool_config (list): A list of dictionaries containing the tool configuration. 
+                            Each dictionary should have a "name" key and a "configuration" key,
+                            where "configuration" is a list of dictionaries containing "name" and "value" keys.
+
+    Returns:
+        str: The SERPAPI_KEY access token.
+
+    Raises:
+        ValueError: If the SerpAPI integration has not been configured.
+    """
+    logger.info("Entering get_serp_api_access_token")
+    SERPAPI_KEY = None
+
+    if tool_config:
+        logger.debug(f"Tool config provided: {tool_config}")
+        serpapi_config = next(
+            (item for item in tool_config if item.get("name") == "serpapi"), None
+        )
+        if serpapi_config:
+            config_map = {
+                item["name"]: item["value"]
+                for item in serpapi_config.get("configuration", [])
+                if item
+            }
+            SERPAPI_KEY = config_map.get("apiKey")
+        else:
+            logger.warning("No 'serpapi' config item found in tool_config.")
+    else:
+        logger.debug("No tool_config provided or it's None.")
+
+    SERPAPI_KEY = SERPAPI_KEY or os.getenv("SERPAPI_KEY")
+    if not SERPAPI_KEY:
+        logger.error("SerpAPI integration is not configured.")
+        raise ValueError(
+            "SerpAPI integration is not configured. Please configure the connection to SerpAPI in Integrations."
+        )
+
+    logger.info("Retrieved SERPAPI_KEY successfully.")
+    return SERPAPI_KEY
+
+
+@assistant_tool
+async def search_google_serpai(
+    query: str,
+    number_of_results: int = 10,
+    offset: int = 0,
+    tool_config: Optional[List[Dict]] = None,
+    as_oq: Optional[str] = None,   # optional terms
+) -> List[str]:
+    """
+    Google search via SerpAPI that returns a *uniform* list of JSON strings.
+    Each item is guaranteed to contain a 'link' key, even when the result
+    originally came from image/news blocks.
+
+    Blocks handled:
+        • organic_results   – keeps SerpAPI structure
+        • inline_images     – maps  source  -> link
+        • news_results      – already has link
+    """
+    logger.info("Entering search_google_serpai")
+    if not query:
+        logger.warning("Empty query string provided.")
+        return []
+
+    cache_key = f"{query}_{number_of_results}_{offset}_{as_oq or ''}"
+    if cached := retrieve_output("search_google_serp", cache_key):
+        logger.info("Cache hit for search_google_serp.")
+        return cached
+
+    SERPAPI_KEY = get_serp_api_access_token(tool_config)
+    base_url    = "https://serpapi.com/search"
+
+    page_size     = number_of_results
+    start_index   = 0 if offset == 0 else offset + 1  # SerpAPI Pagination Mechanism: Uses the start parameter to specify the first result (zero-indexed)
+    all_items: list[dict] = []
+    seen_links:   set[str] = set()     # dedupe across blocks/pages
+
+    # ------------------------------------------------------------------ #
+    # helpers                                                            #
+    # ------------------------------------------------------------------ #
+    def _extract_block_results(block: str, data: list[dict]) -> list[dict]:
+        """Return items from a given block in unified format (must include link)."""
+        mapped: list[dict] = []
+
+        if block == "organic_results":
+            for it in data:
+                link = it.get("link")
+                if link:
+                    mapped.append(it)                   # keep original shape
+        elif block == "inline_images":
+            for it in data:
+                link = it.get("source")                 # image-pack URL
+                if link:
+                    mapped.append({
+                        "title":  it.get("title"),
+                        "link":   link,
+                        "type":  "inline_image",
+                        "source_name": it.get("source_name"),
+                        "thumbnail":  it.get("thumbnail"),
+                    })
+        elif block == "news_results":
+            for it in data:
+                link = it.get("link")
+                if link:
+                    mapped.append(it)                   # already fine
+        return mapped
+    # ------------------------------------------------------------------ #
+
+    async with aiohttp.ClientSession() as session:
+        while len(all_items) < number_of_results:
+            to_fetch = min(page_size, number_of_results - len(all_items))
+            params = {
+                "engine":   "google",
+                "api_key":  SERPAPI_KEY,
+                "q":        query,
+                "num":      to_fetch,
+                "start":    start_index,
+                "location": "United States",
+            }
+            if as_oq:
+                params["as_oq"] = as_oq
+
+            logger.debug(f"SERP API GET → {params}")
+
+            try:
+                async with session.get(base_url, params=params) as resp:
+                    if resp.status != 200:
+                        try:
+                            err = await resp.json()
+                        except Exception:
+                            err = await resp.text()
+                        logger.warning(f"SerpAPI {resp.status=}: {err}")
+                        return [json.dumps({"error": err})]
+                    result = await resp.json()
+            except Exception as e:
+                logger.exception("SerpAPI request failed")
+                return [json.dumps({"error": str(e)})]
+
+            # ------------------ harvest every supported block ------------------
+            page_items: list[dict] = []
+            for block_name in ("organic_results", "inline_images", "news_results"):
+                data = result.get(block_name) or []
+                page_items.extend(_extract_block_results(block_name, data))
+
+            # dedupe & accumulate
+            new_added = 0
+            for it in page_items:
+                link = it["link"]
+                if link not in seen_links:
+                    seen_links.add(link)
+                    all_items.append(it)
+                    new_added += 1
+                    if len(all_items) >= number_of_results:
+                        break
+            logger.debug(f"Added {new_added} items (total={len(all_items)})")
+
+            # stop if Google gave us nothing new
+            if new_added == 0:
+                logger.debug("No more items returned; stopping.")
+                break
+
+            start_index += to_fetch   # next Google results page
+
+    # truncate and serialise
+    all_items = all_items[:number_of_results]
+    serialised = [json.dumps(it) for it in all_items]
+    cache_output("search_google_serp", cache_key, serialised)
+
+    logger.info(f"Returning {len(serialised)} items for '{query}'")
+    return serialised

dhisana/utils/serpapi_local_business_search.py

@@ -0,0 +1,129 @@
+import json
+import os
+import logging
+from typing import Any, Dict, List, Optional
+
+import aiohttp
+
+from dhisana.utils.assistant_tool_tag import assistant_tool
+from dhisana.utils.cache_output_tools import cache_output, retrieve_output
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
+
+
+# ──────────────────────────────────────────────────────────────────────────
+#  Re-use the get_serp_api_access_token helper you already have.
+# ──────────────────────────────────────────────────────────────────────────
+def _normalise_local_result(raw: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Convert a single SerpApi `local_results` item to the standard format.
+    – Falls back to ''/None when fields are absent.
+    – Derives `google_maps_url` from the `links.directions` entry when present,
+      otherwise constructs a CID-based URL from data_cid / place_id.
+    """
+    # ── unpack links ──────────────────────────────────────────────────────────
+    links = raw.get("links") or {}
+    if isinstance(links, list):                     # older payloads: list of dicts
+        links = {x.get("type") or x.get("name"): x.get("link")
+                  for x in links if isinstance(x, dict)}
+
+    # ── compute Google Maps URL ───────────────────────────────────────────────
+    cid = raw.get("data_cid") or raw.get("place_id")
+    google_maps_url = links.get("directions") or (f"https://maps.google.com/?cid={cid}" if cid else "")
+
+    # ── return unified schema ─────────────────────────────────────────────────
+    return {
+        "full_name":            raw.get("title", ""),
+        "organization_name":    raw.get("title", ""),
+        "phone":                raw.get("phone") or raw.get("phone_number") or "",
+        "organization_website": raw.get("website") or links.get("website") or "",
+        "rating":               raw.get("rating"),
+        "reviews":              raw.get("reviews"),
+        "address":              raw.get("address", ""),
+        "google_maps_url":      google_maps_url,
+    }
+
+
+@assistant_tool
+async def search_local_business_serpai(
+    query: str,
+    number_of_results: int = 20,
+    offset: int = 0,
+    tool_config: Optional[List[Dict]] = None,
+    location: Optional[str] = None,
+) -> List[str]:
+    """
+    Fetch Google Local results with SerpApi and return a list of businesses
+    normalised to Dhisana's local-business schema (serialized as JSON strings).
+
+    Args:
+        query:               Search term (e.g. "coffee shops near me").
+        number_of_results:   Total items desired.
+        offset:              Result offset (multiples of 20 on desktop).
+        tool_config:         Optional Dhisana tool-config blob holding the API key.
+        location:            Optional human location string (e.g. "San Jose, CA").
+    """
+    if not query:
+        logger.warning("Empty query string provided.")
+        return []
+
+    # ── cache key
+    cache_key = f"local_{query}_{number_of_results}_{offset}_{location or ''}"
+    cached = retrieve_output("search_local_serp", cache_key)
+    if cached is not None:
+        return cached
+
+    # ── api key
+    from your_module import get_serp_api_access_token  # adjust import if needed
+    SERPAPI_KEY = get_serp_api_access_token(tool_config)
+
+    page_size = 20               # Google Local desktop page size
+    start_index = offset
+    collected: List[Dict[str, Any]] = []
+
+    async with aiohttp.ClientSession() as session:
+        while len(collected) < number_of_results:
+            to_fetch = min(page_size, number_of_results - len(collected))
+
+            params = {
+                "engine": "google_local",
+                "type": "search",
+                "q": query,
+                "api_key": SERPAPI_KEY,
+                "start": start_index,
+                "num": to_fetch,
+            }
+            if location:
+                params["location"] = location
+
+            logger.debug("SerpApi local request params: %s", params)
+
+            try:
+                async with session.get("https://serpapi.com/search", params=params) as resp:
+                    if resp.status != 200:
+                        try:
+                            err = await resp.json()
+                        except Exception:
+                            err = await resp.text()
+                        logger.warning("SerpApi error: %s", err)
+                        return [json.dumps({"error": err})]
+                    payload = await resp.json()
+            except Exception as exc:
+                logger.exception("Request failed.")
+                return [json.dumps({"error": str(exc)})]
+
+            local_results = payload.get("local_results", [])
+            if not local_results:
+                break
+
+            collected.extend(local_results)
+            start_index += to_fetch
+
+    # truncate & normalise
+    normalised = [_normalise_local_result(r) for r in collected[:number_of_results]]
+    serialised = [json.dumps(item) for item in normalised]
+
+    cache_output("search_local_serp", cache_key, serialised)
+    logger.info("Returned %d local businesses for '%s'", len(serialised), query)
+    return serialised