nosible 0.1.8__py3-none-any.whl → 0.2.1__py3-none-any.whl

nosible/nosible_client.py

@@ -2,11 +2,15 @@ import gzip
 import json
 import logging
 import os
+import sys
+import textwrap
 import time
-import traceback
+import types
+import typing
 from collections.abc import Iterator
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from typing import Union
+from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime
+from typing import Union, Optional
 
 import polars as pl
 import requests
@@ -25,8 +29,10 @@ from tenacity import (
 from nosible.classes.result_set import ResultSet
 from nosible.classes.search import Search
 from nosible.classes.search_set import SearchSet
+from nosible.classes.snippet_set import SnippetSet
 from nosible.classes.web_page import WebPageData
 from nosible.utils.json_tools import json_loads
+from nosible.utils.question_builder import _get_question
 from nosible.utils.rate_limiter import PLAN_RATE_LIMITS, RateLimiter, _rate_limited
 
 # Set up a module‐level logger.
@@ -47,37 +53,33 @@ class Nosible:
     llm_api_key : str, optional
         API key for LLM-based query expansions.
     openai_base_url : str
-        Base URL for the OpenAI-compatible LLM API.
-    sentiment_model : str
-        Model to use for sentiment analysis and expansions.
+        Base URL for the OpenAI-compatible LLM API. (default is OpenRouter's API endpoint)
+    sentiment_model : str, optional
+        Model to use for sentiment analysis (default is "openai/gpt-4o").
     timeout : int
         Request timeout for HTTP calls.
-    retries : int, default=5
+    retries : int,
         Number of retry attempts for transient HTTP errors.
-    concurrency : int, default=10
+    concurrency : int,
         Maximum concurrent search requests.
     publish_start : str, optional
-        Earliest publish date filter (ISO formatted date).
+        Start date for when the document was published (ISO format).
     publish_end : str, optional
-        Latest publish date filter (ISO formatted date).
-    include_netlocs : list of str, optional
-        Domains to include.
-    exclude_netlocs : list of str, optional
-        Domains to exclude.
+        End date for when the document was published (ISO format).
     visited_start : str, optional
-        Earliest visit date filter (ISO formatted date).
+        Start date for when the document was visited by NOSIBLE (ISO format).
     visited_end : str, optional
-        Latest visit date filter (ISO formatted date).
+        End date for when the document was visited by NOSIBLE (ISO format).
     certain : bool, optional
-        True if we are 100% sure of the date.
-    include_languages : list of str, optional
-        Language codes to include (Max: 50).
-    exclude_languages : list of str, optional
-        Language codes to exclude (Max: 50).
+        Only include documents where we are 100% sure of the date.
     include_netlocs : list of str, optional
-        Only include results from these domains (Max: 50).
+        List of netlocs (domains) to include in the search. (Max: 50)
     exclude_netlocs : list of str, optional
-        Exclude results from these domains (Max: 50).
+        List of netlocs (domains) to exclude in the search. (Max: 50)
+    include_languages : list of str, optional
+        Languages to include in the search. (Max: 50, ISO 639-1 language codes).
+    exclude_languages : list of str, optional
+        Language codes to exclude in the search (Max: 50, ISO 639-1 language codes).
     include_companies : list of str, optional
         Google KG IDs of public companies to require (Max: 50).
     exclude_companies : list of str, optional
@@ -86,10 +88,6 @@ class Nosible:
         URL hashes of docs to include (Max: 50).
     exclude_docs : list of str, optional
         URL hashes of docs to exclude (Max: 50).
-    openai_base_url : str, optional
-        Base URL for the OpenAI API (default is OpenRouter).
-    sentiment_model : str, optional
-        Model to use for sentiment analysis (default is "openai/gpt-4o").
 
     Notes
     -----
@@ -173,7 +171,7 @@ class Nosible:
             reraise=True,
             stop=stop_after_attempt(self.retries) | stop_after_delay(self.timeout),
             wait=wait_exponential(multiplier=1, min=1, max=10),
-            retry=retry_if_exception_type(Exception),
+            retry=retry_if_exception_type(requests.exceptions.RequestException),
             before_sleep=before_sleep_log(self.logger, logging.WARNING),
         )(self._generate_expansions)
 
@@ -212,6 +210,9 @@ class Nosible:
         n_probes: int = 30,
         n_contextify: int = 128,
         algorithm: str = "hybrid-2",
+        min_similarity: float = None,
+        must_include: list[str] = None,
+        must_exclude: list[str] = None,
         autogenerate_expansions: bool = False,
         publish_start: str = None,
         publish_end: str = None,
@@ -243,38 +244,40 @@ class Nosible:
             List of LLM‐generated expansions.
         sql_filter : list of str, optional
             SQL‐style filter clauses.
-        n_results : int, default=100
+        n_results : int
             Max number of results (max 100).
-        n_probes : int, default=30
+        n_probes : int
             Number of index shards to probe.
-        n_contextify : int, default=128
+        n_contextify : int
             Context window size per result.
-        algorithm : str, default="hybrid-2"
+        algorithm : str
             Search algorithm type.
-        autogenerate_expansions : bool, default=False
+        min_similarity : float
+            Results must have at least this similarity score.
+        must_include : list of str
+            Only results mentioning these strings will be included.
+        must_exclude : list of str
+            Any result mentioning these strings will be excluded.
+        autogenerate_expansions : bool
             Do you want to generate expansions automatically using a LLM?
         publish_start : str, optional
-            Earliest publish date filter (ISO formatted date).
+            Start date for when the document was published (ISO format).
         publish_end : str, optional
-            Latest publish date filter (ISO formatted date).
-        include_netlocs : list of str, optional
-            Domains to include.
-        exclude_netlocs : list of str, optional
-            Domains to exclude.
+            End date for when the document was published (ISO format).
         visited_start : str, optional
-            Earliest visit date filter (ISO formatted date).
+            Start date for when the document was visited by NOSIBLE (ISO format).
         visited_end : str, optional
-            Latest visit date filter (ISO formatted date).
+            End date for when the document was visited by NOSIBLE (ISO format).
         certain : bool, optional
-            True if we are 100% sure of the date.
-        include_languages : list of str, optional
-            Language codes to include (Max: 50).
-        exclude_languages : list of str, optional
-            Language codes to exclude (Max: 50).
+            Only include documents where we are 100% sure of the date.
         include_netlocs : list of str, optional
-            Only include results from these domains (Max: 50).
+            List of netlocs (domains) to include in the search. (Max: 50)
         exclude_netlocs : list of str, optional
-            Exclude results from these domains (Max: 50).
+            List of netlocs (domains) to exclude in the search. (Max: 50)
+        include_languages : list of str, optional
+            Languages to include in the search. (Max: 50, ISO 639-1 language codes).
+        exclude_languages : list of str, optional
+            Language codes to exclude in the search (Max: 50, ISO 639-1 language codes).
         include_companies : list of str, optional
             Google KG IDs of public companies to require (Max: 50).
         exclude_companies : list of str, optional
@@ -297,6 +300,8 @@ class Nosible:
             If neither question nor search are specified
         RuntimeError
             If the response fails in any way.
+        ValueError
+            If `n_results` is greater than 100.
 
         Notes
         -----
@@ -342,6 +347,9 @@ class Nosible:
             n_probes=n_probes,
             n_contextify=n_contextify,
             algorithm=algorithm,
+            min_similarity=min_similarity,
+            must_include=must_include,
+            must_exclude=must_exclude,
             autogenerate_expansions=autogenerate_expansions,
             publish_start=publish_start,
             publish_end=publish_end,
@@ -379,6 +387,9 @@ class Nosible:
         n_probes: int = 30,
         n_contextify: int = 128,
         algorithm: str = "hybrid-2",
+        min_similarity: float = None,
+        must_include: list[str] = None,
+        must_exclude: list[str] = None,
         autogenerate_expansions: bool = False,
         publish_start: str = None,
         publish_end: str = None,
@@ -407,48 +418,50 @@ class Nosible:
             List of expansion terms to use for each search.
         sql_filter : list of str, optional
             SQL-like filters to apply to the search.
-        n_results : int, default=100
+        n_results : int
             Number of results to return per search.
-        n_probes : int, default=30
+        n_probes : int
             Number of probes to use for the search algorithm.
-        n_contextify : int, default=128
+        n_contextify : int
             Context window size for the search.
-        algorithm : str, default="hybrid-2"
+        algorithm : str
             Search algorithm to use.
-        autogenerate_expansions : bool, default=False
-            Do you want to generate expansions automatically using a LLM?
+        min_similarity : float
+            Results must have at least this similarity score.
+        must_include : list of str
+            Only results mentioning these strings will be included.
+        must_exclude : list of str
+            Any result mentioning these strings will be excluded.
+        autogenerate_expansions : bool
+            Do you want to generate expansions automatically using a LLM?.
         publish_start : str, optional
-            Filter results published after this date (ISO formatted date).
+            Start date for when the document was published (ISO format).
         publish_end : str, optional
-            Filter results published before this date (ISO formatted date).
-        include_netlocs : list of str, optional
-            Only include results from these domains.
-        exclude_netlocs : list of str, optional
-            Exclude results from these domains.
+            End date for when the document was published (ISO format).
         visited_start : str, optional
-            Only include results visited after this date (ISO formatted date).
+            Start date for when the document was visited by NOSIBLE (ISO format).
         visited_end : str, optional
-            Only include results visited before this date (ISO formatted date).
+            End date for when the document was visited by NOSIBLE (ISO format).
         certain : bool, optional
-            Only include results with high certainty.
+            Only include documents where we are 100% sure of the date.
+        include_netlocs : list of str, optional
+            List of netlocs (domains) to include in the search. (Max: 50)
+        exclude_netlocs : list of str, optional
+            List of netlocs (domains) to exclude in the search. (Max: 50)
         include_languages : list of str, optional
-            Only include results in these languages (Max: 50).
+            Languages to include in the search. (Max: 50, ISO 639-1 language codes).
         exclude_languages : list of str, optional
-            Exclude results in these languages (Max: 50).
+            Language codes to exclude in the search (Max: 50, ISO 639-1 language codes).
         include_companies : list of str, optional
-            Only include results from these companies (Max: 50).
+            Google KG IDs of public companies to require (Max: 50).
         exclude_companies : list of str, optional
-            Exclude results from these companies (Max: 50).
-        include_netlocs : list of str, optional
-            Only include results from these domains (Max: 50).
-        exclude_netlocs : list of str, optional
-            Exclude results from these domains (Max: 50).
+            Google KG IDs of public companies to forbid (Max: 50).
         include_docs : list of str, optional
-            URL hashes of documents to include (Max: 50).
+            URL hashes of docs to include (Max: 50).
         exclude_docs : list of str, optional
-            URL hashes of documents to exclude (Max: 50).
+            URL hashes of docs to exclude (Max: 50).
 
-        Yields
+        Returns
         ------
         ResultSet or None
             Each completed search’s results, or None on failure.
@@ -461,8 +474,6 @@ class Nosible:
             If both queries and searches are specified.
         TypeError
             If neither queries nor searches are specified.
-        RuntimeError
-            If the response fails in any way.
 
         Notes
         -----
@@ -473,7 +484,10 @@ class Nosible:
         --------
         >>> from nosible import Nosible
         >>> queries = SearchSet(
-        ...     [Search(question="Hedge funds seek to expand into private credit", n_results=5), Search(question="How have the Trump tariffs impacted the US economy?", n_results=5)]
+        ...     [
+        ...         Search(question="Hedge funds seek to expand into private credit", n_results=5),
+        ...         Search(question="How have the Trump tariffs impacted the US economy?", n_results=5),
+        ...     ]
         ... )
         >>> with Nosible() as nos:
         ...     results_list = list(nos.searches(searches=queries))
@@ -484,10 +498,14 @@ class Nosible:
         True True
         True True
         >>> with Nosible() as nos:
-        ...     results_list_str = list(nos.searches(questions=[
-        ...     "What are the terms of the partnership between Microsoft and OpenAI?",
-        ...     "What are the terms of the partnership between Volkswagen and Uber?"
-        ...     ]))
+        ...     results_list_str = list(
+        ...         nos.searches(
+        ...             questions=[
+        ...                 "What are the terms of the partnership between Microsoft and OpenAI?",
+        ...                 "What are the terms of the partnership between Volkswagen and Uber?",
+        ...             ]
+        ...         )
+        ...     )
         >>> nos = Nosible(nosible_api_key="test|xyz")  # doctest: +ELLIPSIS
         >>> nos.searches()  # doctest: +ELLIPSIS
         Traceback (most recent call last):
@@ -515,6 +533,9 @@ class Nosible:
                 n_probes=n_probes,
                 n_contextify=n_contextify,
                 algorithm=algorithm,
+                min_similarity=min_similarity,
+                must_include=must_include,
+                must_exclude=must_exclude,
                 autogenerate_expansions=autogenerate_expansions,
                 publish_start=publish_start,
                 publish_end=publish_end,
@@ -538,7 +559,8 @@ class Nosible:
                     yield future.result()
                 except Exception as e:
                     self.logger.warning(f"Search failed: {e!r}")
-                    yield None
+                    raise
+
         return _run_generator()
 
     @_rate_limited("fast")
@@ -560,6 +582,8 @@ class Nosible:
         ------
         ValueError
             If `n_results` > 100.
+        ValueError
+            If min_similarity is not [0,1].
 
         Examples
         --------
@@ -573,7 +597,7 @@ class Nosible:
         ValueError: Search can not have more than 100 results - Use bulk search instead.
         """
         # --------------------------------------------------------------------------------------------------------------
-        # Setting search params. Individual search will overide Nosible defaults.
+        # Setting search params. Individual search will override Nosible defaults.
         # --------------------------------------------------------------------------------------------------------------
         question = search_obj.question  # No default
         expansions = search_obj.expansions if search_obj.expansions is not None else []  # Default to empty list
@@ -582,7 +606,12 @@ class Nosible:
         n_probes = search_obj.n_probes if search_obj.n_probes is not None else 30
         n_contextify = search_obj.n_contextify if search_obj.n_contextify is not None else 128
         algorithm = search_obj.algorithm if search_obj.algorithm is not None else "hybrid-2"
-        autogenerate_expansions = search_obj.autogenerate_expansions if search_obj.autogenerate_expansions is not None else False
+        min_similarity = search_obj.min_similarity if search_obj.min_similarity is not None else 0
+        must_include = search_obj.must_include if search_obj.must_include is not None else []
+        must_exclude = search_obj.must_exclude if search_obj.must_exclude is not None else []
+        autogenerate_expansions = (
+            search_obj.autogenerate_expansions if search_obj.autogenerate_expansions is not None else False
+        )
         publish_start = search_obj.publish_start if search_obj.publish_start is not None else self.publish_start
         publish_end = search_obj.publish_end if search_obj.publish_end is not None else self.publish_end
         include_netlocs = search_obj.include_netlocs if search_obj.include_netlocs is not None else self.include_netlocs
@@ -603,6 +632,9 @@ class Nosible:
             search_obj.exclude_companies if search_obj.exclude_companies is not None else self.exclude_companies
         )
 
+        if not (0.0 <= min_similarity <= 1.0):
+            raise ValueError(f"Invalid min_simalarity: {min_similarity}.  Must be [0,1].")
+
         # Generate expansions if not provided
         if expansions is None:
             expansions = []
@@ -639,6 +671,9 @@ class Nosible:
             "n_probes": n_probes,
             "n_contextify": n_contextify,
             "algorithm": algorithm,
+            "min_similarity": min_similarity,
+            "must_include": must_include,
+            "must_exclude": must_exclude,
         }
 
         resp = self._post(url="https://www.nosible.ai/search/v1/fast-search", payload=payload)
@@ -699,6 +734,9 @@ class Nosible:
         n_probes: int = 30,
         n_contextify: int = 128,
         algorithm: str = "hybrid-2",
+        min_similarity: float = None,
+        must_include: list[str] = None,
+        must_exclude: list[str] = None,
         autogenerate_expansions: bool = False,
         publish_start: str = None,
         publish_end: str = None,
@@ -728,46 +766,48 @@ class Nosible:
             Optional list of expanded query strings.
         sql_filter : list of str, optional
             Optional SQL WHERE clause filters.
-        n_results : int, default=100
+        n_results : int
             Number of results per query (1,000–10,000).
-        n_probes : int, default=30
+        n_probes : int
             Number of shards to probe.
-        n_contextify : int, default=128
+        n_contextify : int
             Context window size per result.
-        algorithm : str, default="hybrid-2"
+        algorithm : str
             Search algorithm identifier.
-        autogenerate_expansions : bool, default=False
+        min_similarity : float
+            Results must have at least this similarity score.
+        must_include : list of str
+            Only results mentioning these strings will be included.
+        must_exclude : list of str
+            Any result mentioning these strings will be excluded.
+        autogenerate_expansions : bool
             Do you want to generate expansions automatically using a LLM?
         publish_start : str, optional
-            Filter for earliest publish date.
+            Start date for when the document was published (ISO format).
         publish_end : str, optional
-            Filter for latest publish date.
-        include_netlocs : list of str, optional
-            Domains to include.
-        exclude_netlocs : list of str, optional
-            Domains to exclude.
+            End date for when the document was published (ISO format).
         visited_start : str, optional
-            Filter for earliest visit date.
+            Start date for when the document was visited by NOSIBLE (ISO format).
         visited_end : str, optional
-            Filter for latest visit date.
+            End date for when the document was visited by NOSIBLE (ISO format).
         certain : bool, optional
-            True if we are 100% sure of the date.
-        include_languages : list of str, optional
-            Languages to include (Max: 50).
-        exclude_languages : list of str, optional
-            Languages to exclude (Max: 50).
+            Only include documents where we are 100% sure of the date.
         include_netlocs : list of str, optional
-            Only include results from these domains (Max: 50).
+            List of netlocs (domains) to include in the search. (Max: 50)
         exclude_netlocs : list of str, optional
-            Exclude results from these domains (Max: 50).
+            List of netlocs (domains) to exclude in the search. (Max: 50)
+        include_languages : list of str, optional
+            Languages to include in the search. (Max: 50, ISO 639-1 language codes).
+        exclude_languages : list of str, optional
+            Language codes to exclude in the search (Max: 50, ISO 639-1 language codes).
         include_companies : list of str, optional
-            Company IDs to require (Max: 50).
+            Google KG IDs of public companies to require (Max: 50).
         exclude_companies : list of str, optional
-            Company IDs to forbid (Max: 50).
+            Google KG IDs of public companies to forbid (Max: 50).
         include_docs : list of str, optional
-            URL hashes of documents to include (Max: 50).
+            URL hashes of docs to include (Max: 50).
         exclude_docs : list of str, optional
-            URL hashes of documents to exclude (Max: 50).
+            URL hashes of docs to exclude (Max: 50).
         verbose : bool, optional
             Show verbose output, Bulk search will print more information.
 
@@ -786,6 +826,8 @@ class Nosible:
             If neither question nor search are specified.
         RuntimeError
             If the response fails in any way.
+        ValueError
+            If min_similarity is not [0,1].
 
         Notes
         -----
@@ -794,23 +836,21 @@ class Nosible:
 
         Examples
         --------
-        >>> from nosible.classes.search import Search
-        >>> from nosible import Nosible
-        >>> with Nosible(include_netlocs=["bbc.com"]) as nos:  # doctest: +SKIP
-        ...     results = nos.bulk_search(question="Nvidia insiders dump more than $1 billion in stock", n_results=2000)  # doctest: +SKIP
+        >>> from nosible.classes.search import Search  # doctest: +SKIP
+        >>> from nosible import Nosible  # doctest: +SKIP
+        >>> with Nosible(exclude_netlocs=["bbc.com"]) as nos:  # doctest: +SKIP
+        ...     results = nos.bulk_search(question=_get_question(), n_results=2000)  # doctest: +SKIP
         ...     print(isinstance(results, ResultSet))  # doctest: +SKIP
         ...     print(len(results))  # doctest: +SKIP
         True
         2000
-
-        >>> s = Search(question="OpenAI", n_results=1000)  # doctest: +SKIP
+        >>> s = Search(question=_get_question(), n_results=1000)  # doctest: +SKIP
         >>> with Nosible() as nos:  # doctest: +SKIP
         ...     results = nos.bulk_search(search=s)  # doctest: +SKIP
         ...     print(isinstance(results, ResultSet))  # doctest: +SKIP
         ...     print(len(results))  # doctest: +SKIP
         True
         1000
-
         >>> nos = Nosible(nosible_api_key="test|xyz")  # doctest: +SKIP
         >>> nos.bulk_search()  # doctest: +SKIP
         Traceback (most recent call last):
@@ -818,20 +858,18 @@ class Nosible:
         TypeError: Either question or search must be specified
 
         >>> nos = Nosible(nosible_api_key="test|xyz")  # doctest: +SKIP
-        >>> nos.bulk_search(question="foo", search=Search(question="foo"))  # doctest: +SKIP
+        >>> nos.bulk_search(question=_get_question(), search=Search(question=_get_question()))  # doctest: +SKIP
         Traceback (most recent call last):
         ...
         TypeError: Question and search cannot be both specified
-
         >>> nos = Nosible(nosible_api_key="test|xyz")  # doctest: +SKIP
-        >>> nos.bulk_search(question="foo", n_results=100)  # doctest: +SKIP
+        >>> nos.bulk_search(question=_get_question(), n_results=100)  # doctest: +SKIP
         Traceback (most recent call last):
         ...
-        ValueError: Bulk search must have at least 100 results per query; use search() for smaller result sets.
-
+        ValueError: Bulk search must have at least 1000 results per query; use search() for smaller result sets.
         >>> nos = Nosible(nosible_api_key="test|xyz")  # doctest: +SKIP
-        >>> nos.bulk_search(question="foo", n_results=10001)  # doctest: +SKIP
-        Traceback (most recent call last):
+        >>> nos.bulk_search(question=_get_question(), n_results=10001)  # doctest: +SKIP
+        Traceback (most recent call last):  # doctest: +SKIP
         ...
         ValueError: Bulk search cannot have more than 10000 results per query.
         """
@@ -854,8 +892,17 @@ class Nosible:
             n_probes = search.n_probes if search.n_probes is not None else n_probes
             n_contextify = search.n_contextify if search.n_contextify is not None else n_contextify
             algorithm = search.algorithm if search.algorithm is not None else algorithm
-            autogenerate_expansions = search.autogenerate_expansions if search.autogenerate_expansions is not None \
+            min_similarity = search.min_similarity if search.min_similarity is not None else min_similarity
+            min_similarity = min_similarity if min_similarity is not None else 0
+            must_include = search.must_include if search.must_include is not None else must_include
+            must_include = must_include if must_include is not None else []
+            must_exclude = search.must_exclude if search.must_exclude is not None else must_exclude
+            must_exclude = must_exclude if must_exclude is not None else []
+            autogenerate_expansions = (
+                search.autogenerate_expansions
+                if search.autogenerate_expansions is not None
                 else autogenerate_expansions
+            )
             publish_start = search.publish_start if search.publish_start is not None else publish_start
             publish_end = search.publish_end if search.publish_end is not None else publish_end
             include_netlocs = search.include_netlocs if search.include_netlocs is not None else include_netlocs
@@ -876,6 +923,13 @@ class Nosible:
         if autogenerate_expansions is True:
             expansions = self._generate_expansions(question=question)
 
+        must_include = must_include if must_include is not None else []
+        must_exclude = must_exclude if must_exclude is not None else []
+        min_similarity = min_similarity if min_similarity is not None else 0
+
+        if not (0.0 <= min_similarity <= 1.0):
+            raise ValueError(f"Invalid min_simalarity: {min_similarity}.  Must be [0,1].")
+
         # Generate sql_filter if unset
         if sql_filter is None:
             sql_filter = self._format_sql(
@@ -920,6 +974,9 @@ class Nosible:
                 "n_probes": n_probes,
                 "n_contextify": n_contextify,
                 "algorithm": algorithm,
+                "min_similarity": min_similarity,
+                "must_include": must_include,
+                "must_exclude": must_exclude,
             }
             resp = self._post(url="https://www.nosible.ai/search/v1/slow-search", payload=payload)
             try:
@@ -952,20 +1009,139 @@ class Nosible:
             if verbose:
                 self.logger.setLevel(previous_level)
 
+    def answer(
+        self,
+        query: str,
+        n_results: int = 100,
+        min_similarity: float = 0.65,
+        model: Union[str, None] = "google/gemini-2.0-flash-001",
+        show_context: bool = True,
+    ) -> str:
+        """
+        RAG-style question answering: retrieve top `n_results` via `.search()`
+        then answer `query` using those documents as context.
+
+        Parameters
+        ----------
+        query : str
+            The user’s natural-language question.
+        n_results : int
+            How many docs to fetch to build the context.
+        min_similarity : float
+            Results must have at least this similarity score.
+        model : str, optional
+            Which LLM to call to answer your question.
+        show_context : bool, optional
+            Do you want the context to be shown?
+
+        Returns
+        -------
+        str
+            The LLM’s generated answer, grounded in the retrieved docs.
+
+        Raises
+        ------
+        ValueError
+            If no API key is configured for the LLM client.
+        RuntimeError
+            If the LLM call fails or returns an invalid response.
+
+        Examples
+        --------
+        >>> from nosible import Nosible
+        >>> with Nosible() as nos:
+        ...     ans = nos.answer(
+        ...         query="How is research governance and decision-making structured between Google and DeepMind?",
+        ...         n_results=100,
+        ...         show_context=True
+        ...     )  # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+        <BLANKLINE>
+        Doc 1
+        Title: ...
+        >>> print(ans)  # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+        Answer:
+        ...
+        """
+
+        if not self.llm_api_key:
+            raise ValueError("An LLM API key is required for answer().")
+
+        # Retrieve top documents
+        results = self.search(
+            question=query,
+            n_results=n_results,
+            min_similarity=min_similarity,
+        )
+
+        # Build RAG context
+        context = ""
+        pieces: list[str] = []
+        for idx, result in enumerate(results):
+            pieces.append(f"""
+                Doc {idx + 1}
+                Title: {result.title}
+                Similarity Score: {result.similarity * 100:.2f}%
+                URL: {result.url}
+                Content: {result.content}
+                """)
+            context = "\n".join(pieces)
+
+        if show_context:
+            print(textwrap.dedent(context))
+
+        # Craft prompt
+        prompt = (f"""
+            # TASK DESCRIPTION
+
+            You are a helpful assistant.  Use the following context to answer the question.
+            When you use information from a chunk, cite it by referencing its label in square brackets, e.g. [doc3].
+            
+            ## Question
+            {query}
+            
+            ## Context
+            {context}
+            """
+        )
+
+        # Call LLM
+        client = OpenAI(base_url=self.openai_base_url, api_key=self.llm_api_key)
+        try:
+            response = client.chat.completions.create(
+                model = model,
+                messages = [{"role": "user", "content": prompt}],
+            )
+        except Exception as e:
+            raise RuntimeError(f"LLM API error: {e}") from e
+
+        # Validate response shape
+        choices = getattr(response, "choices", None)
+        if not choices or not hasattr(choices[0], "message"):
+            raise RuntimeError(f"Invalid LLM response format: {response!r}")
+
+        # Return the generated text
+        return "Answer:\n" + response.choices[0].message.content.strip()
+
     @_rate_limited("visit")
-    def visit(self, html: str = "", recrawl: bool = False, render: bool = False, url: str = None) -> WebPageData:
+    def visit(
+        self,
+        html: str = "",
+        recrawl: bool = False,
+        render: bool = False,
+        url: str = None
+    ) -> WebPageData:
         """
         Visit a given URL and return a structured WebPageData object for the page.
 
         Parameters
         ----------
-        html : str, default=""
+        html : str
             Raw HTML to process instead of fetching.
-        recrawl : bool, default=False
+        recrawl : bool
             If True, force a fresh crawl.
-        render : bool, default=False
+        render : bool
             If True, allow JavaScript rendering before extraction.
-        url : str, default=None
+        url : str
             The URL to fetch and parse.
 
         Returns
@@ -986,26 +1162,24 @@ class Nosible:
 
         Examples
         --------
-        >>> from nosible import Nosible  # doctest: +SKIP
-        >>> with Nosible() as nos:  # doctest: +SKIP
-        ...     out = nos.visit(url="https://www.dailynewsegypt.com/2023/09/08/g20-and-its-summits/")  # doctest: +SKIP
-        ...     print(isinstance(out, type(WebPageData)))  # doctest: +SKIP
-        ...     print(hasattr(out, "languages"))  # doctest: +SKIP
-        ...     print(hasattr(out, "page"))  # doctest: +SKIP
+        >>> from nosible import Nosible
+        >>> with Nosible() as nos:
+        ...     out = nos.visit(url="https://www.dailynewsegypt.com/2023/09/08/g20-and-its-summits/")
+        ...     print(isinstance(out, WebPageData))
+        ...     print(hasattr(out, "languages"))
+        ...     print(hasattr(out, "page"))
         True
         True
         True
-        >>> with Nosible() as nos:  # doctest: +SKIP
-        ...     out = nos.visit()  # doctest: +SKIP
-        ...     print(isinstance(out, type(WebPageData)))  # doctest: +SKIP
-        ...     print(hasattr(out, "languages"))  # doctest: +SKIP
-        ...     print(hasattr(out, "page"))  # doctest: +SKIP
+        >>> with Nosible() as nos:
+        ...     out = nos.visit()
+        ...     print(isinstance(out, type(WebPageData)))
+        ...     print(hasattr(out, "languages"))
+        ...     print(hasattr(out, "page"))  # doctest: +ELLIPSIS
         Traceback (most recent call last):
         ...
         TypeError: URL must be provided
         """
-
-        # self._enforce("visit")
         if url is None:
             raise TypeError("URL must be provided")
         response = self._post(
@@ -1018,7 +1192,7 @@ class Nosible:
             self.logger.error(f"Failed to parse JSON from response: {e}")
             raise ValueError("Invalid JSON response from server") from e
 
-        if data == {'message': 'Sorry, the URL could not be fetched.'}:
+        if data == {"message": "Sorry, the URL could not be fetched."}:
             raise ValueError("The URL could not be found.")
 
         if "response" not in data:
@@ -1033,12 +1207,84 @@ class Nosible:
             metadata=response_data.get("metadata"),
             page=response_data.get("page"),
             request=response_data.get("request"),
-            snippets=response_data.get("snippets"),
+            snippets=SnippetSet.from_dict(response_data.get("snippets", {})),
             statistics=response_data.get("statistics"),
             structured=response_data.get("structured"),
             url_tree=response_data.get("url_tree"),
         )
 
+    @_rate_limited("fast")
+    def trend(
+        self,
+        query: str,
+        start_date: Optional[str] = None,
+        end_date: Optional[str] = None,
+        sql_filter: Optional[str] = None,
+    ) -> dict:
+        """
+        Extract a trend showing the volume of news surrounding your query.
+
+        Parameters
+        ----------
+        query : str
+            The search term we would like to see a trend for.
+        start_date : str, optional
+            ISO‐format start date (YYYY-MM-DD) of the trend window.
+        end_date : str, optional
+            ISO‐format end date (YYYY-MM-DD) of the trend window.
+        sql_filter : str, optional
+            An optional SQL filter to narrow down the trend query
+
+        Returns
+        -------
+        dict
+            The JSON-decoded trend data returned by the server.
+
+        Examples
+        --------
+        >>> from nosible import Nosible
+        >>> with Nosible() as nos:
+        ...     trends_data = nos.trend("Christmas Shopping", start_date="2005-01-01", end_date="2020-12-31")
+        ...     print(trends_data)  # doctest: +ELLIPSIS
+        {'2005-01-31': ...'2020-12-31': ...}
+        """
+        # Validate dates
+        if start_date is not None:
+            self._validate_date_format(start_date, "start_date")
+        if end_date is not None:
+            self._validate_date_format(end_date, "end_date")
+
+        payload: dict[str, str] = {"query": query}
+
+        if sql_filter is not None:
+            payload["sql_filter"] = sql_filter
+        else:
+            payload["sql_filter"] = "SELECT loc, published FROM engine"
+
+        # Send the POST to the /trend endpoint
+        response = self._post(
+            url="https://www.nosible.ai/search/v1/trend",
+            payload=payload,
+        )
+        # Will raise ValueError on rate-limit or auth errors
+        response.raise_for_status()
+        payload = response.json().get("response", {})
+
+        # if no window requested, return everything
+        if start_date is None and end_date is None:
+            return payload
+
+        # Filter by ISO‐date keys
+        filtered: dict[str, float] = {}
+        for date_str, value in payload.items():
+            if start_date and date_str < start_date:
+                continue
+            if end_date and date_str > end_date:
+                continue
+            filtered[date_str] = value
+
+        return filtered
+
     def version(self) -> str:
         """
         Retrieve the current version information for the Nosible API.
@@ -1097,10 +1343,6 @@ class Nosible:
 
         Raises
         ------
-        ValueError
-            If the API returns an unexpected message.
-        requests.HTTPError
-            If the HTTP request fails.
 
         Examples
         --------
@@ -1121,10 +1363,13 @@ class Nosible:
                 return False
             if msg == "The URL could not be retrieved.":
                 return False
+            # If we reach here, the response is unexpected
+            return False
         except requests.HTTPError:
             return False
         except:
             return False
+
     def preflight(self, url: str = None) -> str:
         """
         Run a preflight check for crawling/preprocessing on a URL.
@@ -1180,40 +1425,47 @@ class Nosible:
 
         Examples
         --------
-        >>> nos = Nosible(nosible_api_key="test|xyz")  # doctest: +SKIP
-        >>> print(nos.get_rate_limits())  # doctest: +SKIP
-        Free (Your current plan)
-        | Endpoint    | Per Month | Per Day | Per Minute |
-        | ----------- | --------- | ------- | ---------- |
-        | Fast Search |     3 000 |     100 |         10 |
-        | URL Visits  |       300 |      10 |          1 |
-        | Slow Search |       300 |      10 |          1 |
-
-        Basic
-        | Endpoint    | Per Month | Per Day | Per Minute |
+        >>> nos = Nosible(nosible_api_key="test|xyz")
+        >>> print(nos.get_rate_limits())  # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+        Below are the rate limits for all NOSIBLE plans.
+        To upgrade your package, visit https://www.nosible.ai/products.
+        <BLANKLINE>
+        Unless otherwise indicated, bulk searches are limited to one-at-a-time per API key.
+        <BLANKLINE>
+        Free: (Your current plan)
+        | Endpoint    | Per Month | Per Minute | Effective CPM |
+        | ----------- | --------- | ---------- | ------------- |
+        | Search      |      3000 |         60 |         $4.00 |
+        | URL Visits  |       300 |         60 |         $4.00 |
+        | Bulk Search |       300 |         60 |         $4.00 |
+        <BLANKLINE>
+        Basic ($49p/m):
+        | Endpoint    | Per Month | Per Minute | Effective CPM |
         ...
         """
         # Human-friendly plan names
         display = {
             "test": "Free",
-            "basic": "Basic",
-            "pro": "Pro",
-            "pro+": "Pro+",
-            "bus": "Business",
-            "bus+": "Business+",
-            "ent": "Enterprise",
+            "basic": "Basic ($49p/m)",
+            "pro": "Pro ($199p/m)",
+            "pro+": "Pro+ ($799p/m)",
+            "bus": "Business ($3999p/m)",
+            "bus+": "Business+ ($7499p/m)",
+            "ent": "Enterprise ($14999p/m)",
         }
 
         # Human-friendly endpoint names
-        endpoint_name = {"fast": "Fast Search", "visit": "URL Visits", "slow": "Bulk Search"}
+        endpoint_name = {"fast": "Search", "visit": "URL Visits", "slow": "Bulk Search"}
 
         out = [
             "Below are the rate limits for all NOSIBLE plans.",
             "To upgrade your package, visit https://www.nosible.ai/products.\n",
+            "Unless otherwise indicated, bulk searches are limited to one-at-a-time per API key.\n"
         ]
 
         user_plan = self._get_user_plan()
         current_plan = ""
+        cpm_counter = 4.0
 
         # Preserve the order you care about:
         for plan in ["test", "basic", "pro", "pro+", "bus", "bus+", "ent"]:
@@ -1222,17 +1474,19 @@ class Nosible:
                 current_plan = " (Your current plan)"
 
             out.append(f"{name}:{current_plan}")
-            out.append("| Endpoint    | Per Month | Per Day | Per Minute |")
-            out.append("| ----------- | --------- | ------- | ---------- |")
+            out.append("| Endpoint    | Per Month | Per Minute | Effective CPM |")
+            out.append("| ----------- | --------- | ---------- | ------------- |")
 
             for ep in ["fast", "visit", "slow"]:
                 buckets = PLAN_RATE_LIMITS[plan][ep]
                 # Find minute & day
                 minute = next(limit for limit, i in buckets if i == 60)
-                day = next(limit for limit, i in buckets if i == 24 * 3600)
-                month = day * 30
-                out.append(f"| {endpoint_name[ep]:<11} | {month:>9} | {day:>7} | {minute:>10} |")
+                month = next(limit for limit, i in buckets if i == 24 * 3600 * 30)
+                cpm = f"${cpm_counter:.2f}"
+
+                out.append(f"| {endpoint_name[ep]:<11} | {month:>9} | {minute:>10} | {cpm:>13} |")
 
+            cpm_counter = cpm_counter - 0.5
             out.append("")  # Blank line
             current_plan = ""
 
@@ -1243,10 +1497,6 @@ class Nosible:
         Close the Nosible client, shutting down the HTTP session
         and thread pool to release network and threading resources.
 
-        Returns
-        -------
-        None
-
         Examples
         --------
         >>> from nosible import Nosible
@@ -1292,6 +1542,8 @@ class Nosible:
             If the user API key is invalid.
         ValueError
             If the user hits their rate limit.
+        ValueError
+            If the user is making too many concurrent searches.
         ValueError
             If an unexpected error occurs.
         ValueError
@@ -1319,12 +1571,17 @@ class Nosible:
             content_type = response.headers.get("Content-Type", "")
             if content_type.startswith("application/json"):
                 body = response.json()
+                if isinstance(body, list):
+                    body = body[0]  # NOSIBLE returns a list of errors
+                print(body)
                 if body.get("type") == "string_too_short":
                     raise ValueError("Your API key is not valid: Too Short.")
             else:
                 raise ValueError("You made a bad request.")
         if response.status_code == 429:
             raise ValueError("You have hit your rate limit.")
+        if response.status_code == 409:
+            raise ValueError("Too many concurrent searches.")
         if response.status_code == 500:
             raise ValueError("An unexpected error occurred.")
         if response.status_code == 502:
@@ -1354,16 +1611,16 @@ class Nosible:
 
         Examples
         --------
-        >>> nos = Nosible(nosible_api_key="test+|xyz")  # doctest: +SKIP
+        >>> nos = Nosible(nosible_api_key="test+|xyz")  # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
         Traceback (most recent call last):
         ...
-        ValueError: test+ is not a valid plan prefix, your API key is invalid.
+        ValueError: Your API key is not valid: test+ is not a valid plan prefix.
         """
         # Split off anything after the first '|'
         prefix = (self.nosible_api_key or "").split("|", 1)[0]
 
-        # Map prefixes -> human-friendly plan names
-        plans = {"test", "basic", "pro", "pro+", "bus", "bus+", "ent"}
+        # Map prefixes -> plan names
+        plans = {"test", "basic", "pro", "pro+", "bus", "bus+", "ent", "chat"}
 
         if prefix not in plans:
             raise ValueError(f"Your API key is not valid: {prefix} is not a valid plan prefix.")
@@ -1393,11 +1650,10 @@ class Nosible:
 
         Examples
         --------
-
-        >>> from nosible import Nosible  # doctest: +SKIP
-        >>> nos = Nosible(llm_api_key=None)  # doctest: +SKIP
-        >>> nos.llm_api_key = None  # doctest: +SKIP
-        >>> nos._generate_expansions("anything")  # doctest: +SKIP
+        >>> from nosible import Nosible
+        >>> nos = Nosible(llm_api_key=None)
+        >>> nos.llm_api_key = None
+        >>> nos._generate_expansions("anything")  # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
         Traceback (most recent call last):
         ...
         ValueError: LLM API key is required for generating expansions.
@@ -1486,6 +1742,49 @@ class Nosible:
         self.logger.debug(f"Successful expansions: {expansions}")
         return expansions
 
+    @staticmethod
+    def _validate_date_format(string: str, name: str):
+        """
+        Check that a date string is valid ISO format (YYYY-MM-DD or full ISO timestamp).
+
+        Parameters
+        ----------
+        string : str
+            The date string to validate.
+        name : str
+            The name of the parameter being validated, used in the error message.
+
+        Raises
+        ------
+        ValueError
+            If `string` is not a valid ISO 8601 date. Error message will include
+            the `name` and the offending string.
+                Examples
+        --------
+        >>> # valid date-only format
+        >>> Nosible._validate_date_format("2023-12-31", "publish_start")
+        >>> # valid full timestamp
+        >>> Nosible._validate_date_format("2023-12-31T15:30:00", "visited_end")
+        >>> # invalid month
+        >>> Nosible._validate_date_format("2023-13-01", "publish_end")
+        Traceback (most recent call last):
+            ...
+        ValueError: Invalid date for 'publish_end': '2023-13-01'.  Expected ISO format 'YYYY-MM-DD'.
+        >>> # wrong separator
+        >>> Nosible._validate_date_format("2023/12/31", "visited_start")
+        Traceback (most recent call last):
+            ...
+        ValueError: Invalid date for 'visited_start': '2023/12/31'.  Expected ISO format 'YYYY-MM-DD'.
+        """
+        try:
+            # datetime.fromisoformat accepts both YYYY-MM-DD and full timestamps
+            parsed = datetime.fromisoformat(string)
+        except Exception:
+            raise ValueError(
+                f"Invalid date for '{name}': {string!r}.  "
+                "Expected ISO format 'YYYY-MM-DD'."
+            )
+
     def _format_sql(
         self,
         publish_start: str = None,
@@ -1508,35 +1807,31 @@ class Nosible:
         Parameters
         ----------
         publish_start : str, optional
-            Earliest published date filter.
+            Start date for when the document was published (ISO format).
         publish_end : str, optional
-            Latest published date filter.
-        include_netlocs : list of str, optional
-            Domains to whitelist.
-        exclude_netlocs : list of str, optional
-            Domains to blacklist.
+            End date for when the document was published (ISO format).
         visited_start : str, optional
-            Earliest visit date filter.
+            Start date for when the document was visited by NOSIBLE (ISO format).
         visited_end : str, optional
-            Latest visit date filter.
+            End date for when the document was visited by NOSIBLE (ISO format).
         certain : bool, optional
-            True if we are 100% sure of the date.
-        include_languages : list of str, optional
-            Languages to include (Max: 50).
-        exclude_languages : list of str, optional
-            Languages to exclude (Max: 50).
+            Only include documents where we are 100% sure of the date.
         include_netlocs : list of str, optional
-            Only include results from these domains (Max: 50).
+            List of netlocs (domains) to include in the search. (Max: 50)
         exclude_netlocs : list of str, optional
-            Exclude results from these domains (Max: 50).
+            List of netlocs (domains) to exclude in the search. (Max: 50)
+        include_languages : list of str, optional
+            Languages to include in the search. (Max: 50, ISO 639-1 language codes).
+        exclude_languages : list of str, optional
+            Language codes to exclude in the search (Max: 50, ISO 639-1 language codes).
         include_companies : list of str, optional
-            Public Company Google KG IDs to require (Max: 50).
+            Google KG IDs of public companies to require (Max: 50).
         exclude_companies : list of str, optional
-            Public Company Google KG IDs to forbid (Max: 50).
+            Google KG IDs of public companies to forbid (Max: 50).
         include_docs : list of str, optional
-            URL hashes of documents to include (Max: 50).
+            URL hashes of docs to include (Max: 50).
         exclude_docs : list of str, optional
-            URL hashes of documents to exclude (Max: 50).
+            URL hashes of docs to exclude (Max: 50).
 
         Returns
         -------
@@ -1545,23 +1840,31 @@ class Nosible:
 
         Raises
         ------
-
         ValueError
             If more than 50 items in a filter are given.
         """
+        for name, value in [
+            ("publish_start", publish_start),
+            ("publish_end", publish_end),
+            ("visited_start", visited_start),
+            ("visited_end", visited_end),
+        ]:
+            if value is not None:
+                self._validate_date_format(string=value, name=name)
+
         # Validate list lengths
-        for name, lst in [
-            ('include_netlocs', include_netlocs),
-            ('exclude_netlocs', exclude_netlocs),
-            ('include_languages', include_languages),
-            ('exclude_languages', exclude_languages),
-            ('include_companies', include_companies),
-            ('exclude_companies', exclude_companies),
-            ('include_docs', include_docs),
-            ('exclude_docs', exclude_docs),
+        for name, value in [
+            ("include_netlocs", include_netlocs),
+            ("exclude_netlocs", exclude_netlocs),
+            ("include_languages", include_languages),
+            ("exclude_languages", exclude_languages),
+            ("include_companies", include_companies),
+            ("exclude_companies", exclude_companies),
+            ("include_docs", include_docs),
+            ("exclude_docs", exclude_docs),
         ]:
-            if lst is not None and len(lst) > 50:
-                raise ValueError(f"Too many items for '{name}' filter ({len(lst)}); maximum allowed is 50.")
+            if value is not None and len(value) > 50:
+                raise ValueError(f"Too many items for '{name}' filter ({len(value)}); maximum allowed is 50.")
 
         sql = ["SELECT loc FROM engine"]
         clauses: list[str] = []
@@ -1595,10 +1898,10 @@ class Nosible:
             variants = set()
             for n in include_netlocs:
                 variants.add(n)
-                if n.startswith('www.'):
+                if n.startswith("www."):
                     variants.add(n[4:])
                 else:
-                    variants.add('www.' + n)
+                    variants.add("www." + n)
             in_list = ", ".join(f"'{v}'" for v in sorted(variants))
             clauses.append(f"netloc IN ({in_list})")
 
@@ -1607,10 +1910,10 @@ class Nosible:
             variants = set()
             for n in exclude_netlocs:
                 variants.add(n)
-                if n.startswith('www.'):
+                if n.startswith("www."):
                     variants.add(n[4:])
                 else:
-                    variants.add('www.' + n)
+                    variants.add("www." + n)
             ex_list = ", ".join(f"'{v}'" for v in sorted(variants))
             clauses.append(f"netloc NOT IN ({ex_list})")
 
@@ -1703,7 +2006,7 @@ class Nosible:
         except Exception:
             return False
 
-    def __enter__(self):
+    def __enter__(self) -> "Nosible":
         """
         Enter the context manager, returning this client instance.
 
@@ -1714,32 +2017,46 @@ class Nosible:
         """
         return self
 
-    def __exit__(self, exc_type: type, exc: Exception, tb: traceback):
+    def __exit__(
+        self,
+        _exc_type: typing.Optional[type[BaseException]],
+        _exc_val: typing.Optional[BaseException],
+        _exc_tb: typing.Optional[types.TracebackType],
+    ) -> typing.Optional[bool]:
         """
-        Exit the context manager, ensuring cleanup of resources.
+        Always clean up (self.close()), but let exceptions propagate.
+        Return True only if you really want to suppress an exception.
 
         Parameters
         ----------
-        exc_type : type or None
-            Exception type if raised.
-        exc : Exception or None
-            Exception instance if raised.
-        tb : traceback or None
-            Traceback if exception was raised.
+        _exc_type : Optional[type[BaseException]]
+            The type of the exception raised, if any.
+        _exc_val : Optional[BaseException]
+            The exception instance, if any.
+        _exc_tb : Optional[types.TracebackType]
+            The traceback object, if any.
 
         Returns
         -------
-        None
+        Optional[bool]
+            False to propagate exceptions, True to suppress them.
         """
-        self.close()
+        try:
+            self.close()
+        except Exception as cleanup_err:
+            # optional: log or re-raise, but don’t hide the original exc
+            print(f"Cleanup failed: {cleanup_err!r}")
+        # Return False (or None) => exceptions inside the with‐block are re-raised.
+        return False
 
     def __del__(self):
         """
         Destructor to ensure resources are cleaned up if not explicitly closed.
 
-        Returns
-        -------
-        None
         """
-        # Ensure it's called
-        self.close()
+        # Only close if interpreter is fully alive
+        if not getattr(sys, "is_finalizing", lambda: False)():
+            try:
+                self.close()
+            except Exception:
+                pass