nosible 0.3.6__tar.gz → 0.3.10__tar.gz

{nosible-0.3.6/src/nosible.egg-info → nosible-0.3.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nosible
-Version: 0.3.6
+Version: 0.3.10
 Summary: Python client for the NOSIBLE Search API
 Home-page: https://github.com/NosibleAI/nosible-py
 Author: Stuart Reid, Matthew Dicks, Richard Taylor, Gareth Warburton
@@ -31,7 +31,7 @@ Requires-Dist: polars
 Requires-Dist: duckdb
 Requires-Dist: openai
 Requires-Dist: tantivy
-Requires-Dist: pyrate-limiter
+Requires-Dist: pyrate-limiter<4
 Requires-Dist: tenacity
 Requires-Dist: cryptography
 Requires-Dist: pyarrow

{nosible-0.3.6 → nosible-0.3.10}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "nosible"
-version = "0.3.6"
+version = "0.3.10"
 description = "Python client for the NOSIBLE Search API"
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.9"
@@ -16,7 +16,7 @@ dependencies = [
   "duckdb",
   "openai",
   "tantivy",
-  "pyrate-limiter",
+  "pyrate-limiter<4",
   "tenacity",
   "cryptography",
   "pyarrow",
@@ -61,5 +61,5 @@ dev-dependencies = [
   "pytest-doctestplus",
   "pytest-xdist",
   "urllib3==1.26.15",
-  "hishel",
+  "hishel[async]",
 ]

{nosible-0.3.6 → nosible-0.3.10}/src/nosible/classes/result.py

@@ -36,6 +36,8 @@ class Result:
         The author of the content.
     content : str, optional
         The main content or body of the search result.
+    best_chunk : str, optional
+        The best snippet of text that matches your question from the search result.
     language : str, optional
         The language code of the content (e.g., 'en' for English).
     similarity : float, optional
@@ -105,6 +107,8 @@ class Result:
     """The author of the content."""
     content: str | None = None
     """The main content or body of the search result."""
+    best_chunk: str | None = None
+    """The best snippet of text that matches your question from the search result."""
     language: str | None = None
     """The language code of the content (e.g., 'en' for English)."""
     similarity: float | None = None
@@ -150,23 +154,14 @@ class Result:
         >>> result = Result(title="Example Domain", similarity=0.9876)
         >>> print(str(result))
           0.99 | Example Domain
-        >>> result = Result(title=None, similarity=None)
-        >>> print(str(result))
-        {
-            "url": null,
-            "title": null,
-            "description": null,
-            "netloc": null,
-            "published": null,
-            "visited": null,
-            "author": null,
-            "content": null,
-            "language": null,
-            "similarity": null,
-            "url_hash": null
-        }
         """
-        return print_dict(self.to_dict())
+        # Get the full dictionary
+        data = self.to_dict()
+
+        # Create a new dictionary excluding keys where the value is None
+        clean_data = {k: v for k, v in data.items() if v is not None}
+
+        return print_dict(clean_data)
 
     def __getitem__(self, key: str) -> str | float | bool | None:
         """
@@ -519,6 +514,11 @@ class Result:
         try:
             from nosible import Search
 
+            # Exclude the original doc from the new search.
+            exclude_docs_list = list(exclude_docs) if exclude_docs else []
+            if self.url_hash and self.url_hash not in exclude_docs_list:
+                exclude_docs_list.append(self.url_hash)
+
             s = Search(
                 question=self.title,
                 expansions=[],
@@ -537,7 +537,7 @@ class Result:
                 include_companies=include_companies,
                 exclude_companies=exclude_companies,
                 include_docs=include_docs,
-                exclude_docs=exclude_docs,
+                exclude_docs=exclude_docs_list,
                 brand_safety=brand_safety,
                 language=language,
                 continent=continent,

{nosible-0.3.6 → nosible-0.3.10}/src/nosible/classes/result_set.py

@@ -54,6 +54,7 @@ class ResultSet(Iterator[Result]):
         "visited",
         "author",
         "content",
+        "best_chunk",
         "language",
         "similarity",
         "url_hash",
@@ -1004,6 +1005,7 @@ class ResultSet(Iterator[Result]):
                     visited=row.get("visited"),
                     author=row.get("author"),
                     content=row.get("content"),
+                    best_chunk=row.get("best_chunk"),
                     language=row.get("language"),
                     similarity=row.get("similarity"),
                     url_hash=row.get("url_hash"),
@@ -1113,6 +1115,7 @@ class ResultSet(Iterator[Result]):
                     visited=row.get("visited"),
                     author=row.get("author"),
                     content=row.get("content"),
+                    best_chunk=row.get("best_chunk"),
                     language=row.get("language"),
                     similarity=row.get("semantics", {}).get("similarity", row.get("similarity")),
                     url_hash=row.get("url_hash"),
@@ -1212,6 +1215,7 @@ class ResultSet(Iterator[Result]):
                                 visited=data.get("visited"),
                                 author=data.get("author"),
                                 content=data.get("content"),
+                                best_chunk=data.get("best_chunk"),
                                 language=data.get("language"),
                                 similarity=data.get("similarity"),
                                 url_hash=data.get("url_hash"),
@@ -1449,6 +1453,7 @@ class ResultSet(Iterator[Result]):
                     visited=d.get("visited"),
                     author=d.get("author"),
                     content=d.get("content"),
+                    best_chunk=d.get("best_chunk"),
                     language=d.get("language"),
                     similarity=d.get("similarity", d.get("semantics", {}).get("similarity")),
                     url_hash=d.get("url_hash"),

{nosible-0.3.6 → nosible-0.3.10}/src/nosible/classes/snippet.py

@@ -30,18 +30,13 @@ class Snippet:
         The words in the snippet.
     links : list or None
         List of links associated with the snippet.
-    companies : list or None
-        List of companies mentioned in the snippet.
-
 
     Examples
     --------
     >>> snippet = Snippet(content="Example snippet", language="en")
     >>> print(snippet.content)
     Example snippet
-
     """
-
     content: str = field(default=None, repr=True, compare=True)
     """The text content of the snippet."""
     images: list = field(default=None, repr=True, compare=False)
@@ -62,8 +57,6 @@ class Snippet:
     """The words in the snippet."""
     links: list = field(default=None, repr=False, compare=False)
     """List of links associated with the snippet."""
-    companies: list = field(default=None, repr=False, compare=False)
-    """List of companies mentioned in the snippet."""
 
     def __str__(self):
         """

{nosible-0.3.6 → nosible-0.3.10}/src/nosible/classes/web_page.py

@@ -40,8 +40,6 @@ class WebPageData:
     {'description': 'Example'}
     """
 
-    companies: list = None
-    """A list of companies mentioned in the webpage, if applicable. (GKIDS)"""
     full_text: str = None
     """The full text content of the webpage."""
     languages: dict = None

{nosible-0.3.6 → nosible-0.3.10}/src/nosible/nosible_client.py

@@ -29,7 +29,7 @@ from nosible.classes.search_set import SearchSet
 from nosible.classes.snippet_set import SnippetSet
 from nosible.classes.web_page import WebPageData
 from nosible.utils.json_tools import json_loads
-from nosible.utils.rate_limiter import PLAN_RATE_LIMITS, RateLimiter, _rate_limited
+from nosible.utils.rate_limiter import RateLimiter, _rate_limited
 
 # Set up a module‐level logger.
 logger = logging.getLogger(__name__)
@@ -202,11 +202,6 @@ class Nosible:
         logging.getLogger("httpx").setLevel(logging.WARNING)
         logging.getLogger("httpcore").setLevel(logging.WARNING)
 
-        self._limiters = {
-            endpoint: [RateLimiter(calls, period) for calls, period in buckets]
-            for endpoint, buckets in PLAN_RATE_LIMITS[self._get_user_plan()].items()
-        }
-
         # Define retry decorator
         self._post = retry(
             reraise=True,
@@ -230,7 +225,34 @@ class Nosible:
         self._executor = ThreadPoolExecutor(max_workers=self.concurrency)
 
         # Headers
-        self.headers = {"Accept-Encoding": "gzip", "Content-Type": "application/json", "api-key": self.nosible_api_key}
+        self.headers = {
+            "Accept-Encoding": "gzip",
+            "Content-Type": "application/json",
+            "api-key": self.nosible_api_key
+        }
+
+        # Wrap _get_limits with retry.
+        self._get_limits = retry(
+            reraise=True,
+            stop=stop_after_attempt(self.retries) | stop_after_delay(self.timeout),
+            wait=wait_exponential(multiplier=1, min=1, max=20),
+            retry=retry_if_exception_type(httpx.RequestError),
+            before_sleep=before_sleep_log(self.logger, logging.WARNING),
+        )(self._get_limits)
+
+        raw_limits = self._get_limits()
+
+        # Map API query_type -> your decorator endpoint keys
+        mapped_limits = {
+            "fast": raw_limits.get("fast", []),
+            "bulk": raw_limits.get("slow", []),
+            "scrape-url": raw_limits.get("visit", []),
+        }
+
+        self._limiters = {
+            endpoint: [RateLimiter(calls, period) for calls, period in buckets]
+            for endpoint, buckets in mapped_limits.items()
+        }
 
         # Filters
         self.publish_start = publish_start
@@ -1522,7 +1544,6 @@ class Nosible:
 
         response_data = data["response"]
         return WebPageData(
-            companies=response_data.get("companies"),
             full_text=response_data.get("full_text"),
             languages=response_data.get("languages"),
             metadata=response_data.get("metadata"),
@@ -1603,7 +1624,6 @@ class Nosible:
 
         return filtered
 
-
     def close(self):
         """
         Close the Nosible client, shutting down the HTTP session
@@ -1703,41 +1723,52 @@ class Nosible:
 
         return response
 
-    def _get_user_plan(self) -> str:
+    def _get_limits(self) -> dict[str, list[tuple[int, float]]]:
+        """
+        TODO
         """
-        Determine the user's subscription plan from the API key.
+        url = "https://www.nosible.ai/search/v2/limits"
+        resp = self._session.get(
+            url=url,
+            headers=self.headers,
+            timeout=self.timeout,
+            follow_redirects=True,
+        )
 
-        The `nosible_api_key` is expected to start with a plan prefix followed by
-        a pipe (`|`) and any additional data. This method splits on the first
-        pipe character, validates the prefix against supported plans, and returns it.
+        if resp.status_code == 401:
+            raise ValueError("Your API key is not valid.")
+        if resp.status_code == 429:
+            raise ValueError("You have hit your rate limit.")
+        if resp.status_code == 409:
+            raise ValueError("Too many concurrent searches.")
+        if resp.status_code == 502:
+            raise ValueError("NOSIBLE is currently restarting.")
+        if resp.status_code == 504:
+            raise ValueError("NOSIBLE is currently overloaded.")
 
-        Returns
-        -------
-        str
-            The plan you are currently on.
+        resp.raise_for_status()
 
-        Raises
-        ------
-        ValueError
-            If the extracted prefix is not one of the recognized plan names.
+        try:
+            data = resp.json()
+        except Exception as e:
+            raise ValueError("Invalid JSON response from /limits") from e
 
-        Examples
-        --------
-        >>> nos = Nosible(nosible_api_key="test+|xyz")  # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
-        Traceback (most recent call last):
-        ...
-        ValueError: Your API key is not valid: test+ is not a valid plan prefix.
-        """
-        # Split off anything after the first '|'
-        prefix = (self.nosible_api_key or "").split("|", 1)[0]
+        limits_list = data.get("limits")
+        if not isinstance(limits_list, list):
+            raise ValueError(f"Invalid /limits response shape: {data!r}")
+
+        grouped: dict[str, list[tuple[int, float]]] = {}
+        for item in limits_list:
+            query_type = item.get("query_type")
+            duration = item.get("duration_seconds")
+            limit = item.get("limit")
 
-        # Map prefixes -> plan names
-        plans = {"test", "self", "basic", "pro", "pro+", "bus", "bus+", "ent", "chat", "cons", "stup", "busn", "prod"}
+            if query_type is None or duration is None or limit is None:
+                raise ValueError(f"Invalid limit entry: {item!r}")
 
-        if prefix not in plans:
-            raise ValueError(f"Your API key is not valid: {prefix} is not a valid plan prefix.")
+            grouped.setdefault(str(query_type), []).append((int(limit), float(duration)))
 
-        return prefix
+        return grouped
 
     def _generate_expansions(self, question: Union[str, Search]) -> list:
         """
@@ -2039,13 +2070,13 @@ class Nosible:
 
         if include_docs:
             # Assume these are URL hashes, e.g. "ENNmqkF1mGNhVhvhmbUEs4U2"
-            doc_hashes = ", ".join(f"'{doc}'" for doc in include_docs)
-            clauses.append(f"doc_hash IN ({doc_hashes})")
+            docs = ", ".join(f"'{doc}'" for doc in include_docs)
+            clauses.append(f"doc IN ({docs})")
 
         if exclude_docs:
             # Assume these are URL hashes, e.g. "ENNmqkF1mGNhVhvhmbUEs4U2"
-            doc_hashes = ", ".join(f"'{doc}'" for doc in exclude_docs)
-            clauses.append(f"doc_hash NOT IN ({doc_hashes})")
+            docs = ", ".join(f"'{doc}'" for doc in exclude_docs)
+            clauses.append(f"doc NOT IN ({docs})")
 
         # Join everything
         if clauses:
@@ -2092,7 +2123,7 @@ class Nosible:
             "netloc",
             "language",
             "companies"
-            "doc_hash",
+            "doc",
         ]
         import polars as pl  # Lazy import
 

{nosible-0.3.6 → nosible-0.3.10}/src/nosible/utils/rate_limiter.py

@@ -4,81 +4,10 @@ import time
 
 from pyrate_limiter import Limiter, Rate
 from pyrate_limiter.buckets.in_memory_bucket import InMemoryBucket
-from pyrate_limiter.exceptions import BucketFullException
+from pyrate_limiter.exceptions import BucketFullException, LimiterDelayException
 
 log = logging.getLogger(__name__)
 
-PLAN_RATE_LIMITS = {
-    "test": {
-        # Per minute limit, then per month.
-        "scrape-url": [(60, 60), (300, 24 * 3600 * 30)],
-        "bulk": [(60, 60), (300, 24 * 3600 * 30)],
-        "fast": [(60, 60), (3000, 24 * 3600 * 30)],
-    },
-    "basic": {
-        "scrape-url": [(60, 60), (1400, 24 * 3600 * 30)],
-        "bulk": [(60, 60), (1400, 24 * 3600 * 30)],
-        "fast": [(60, 60), (14_000, 24 * 3600 * 30)],
-    },
-    "pro": {
-        "scrape-url": [(60, 60), (6700, 24 * 3600 * 30)],
-        "bulk": [(60, 60), (6700, 24 * 3600 * 30)],
-        "fast": [(60, 60), (67_000, 24 * 3600 * 30)],
-    },
-    "pro+": {
-        "scrape-url": [(60, 60), (32_000, 24 * 3600 * 30)],
-        "bulk": [(60, 60), (32_000, 24 * 3600 * 30)],
-        "fast": [(60, 60), (320_000, 24 * 3600 * 30)],
-    },
-    "bus": {
-        "scrape-url": [(60, 60), (200_000, 24 * 3600 * 30)],
-        "bulk": [(60, 60), (200_000, 24 * 3600 * 30)],
-        "fast": [(60, 60), (2_000_000, 24 * 3600 * 30)],
-    },
-    "bus+": {
-        "scrape-url": [(60, 60), (500_000, 24 * 3600 * 30)],
-        "bulk": [(60, 60), (500_000, 24 * 3600 * 30)],
-        "fast": [(120, 60), (5_000_000, 24 * 3600 * 30)],
-    },
-    "ent": {
-        "scrape-url": [(60, 60), (1_500_000, 24 * 3600 * 30)],
-        "bulk": [(60, 60), (1_500_000, 24 * 3600 * 30)],
-        "fast": [(360, 60), (15_000_000, 24 * 3600 * 30)],
-    },
-    "prod": {
-        "scrape-url": [(60, 60), (1_500_000, 24 * 3600 * 30)],
-        "bulk": [(60, 60), (1_500_000, 24 * 3600 * 30)],
-        "fast": [(360, 60), (15_000_000, 24 * 3600 * 30)],
-    },
-    # This plan is used for testing in the package
-    "chat": {
-        "scrape-url": [(60, 60), (1_500_000, 24 * 3600 * 30)],
-        "bulk": [(60, 60), (1_500_000, 24 * 3600 * 30)],
-        "fast": [(360, 60), (15_000_000, 24 * 3600 * 30)],
-    },
-    "self": {
-        "scrape-url": [(6000, 60), (1_500_000, 24 * 3600 * 30)],
-        "bulk": [(6000, 60), (1_500_000, 24 * 3600 * 30)],
-        "fast": [(36_000, 60), (15_000_000, 24 * 3600 * 30)],
-    },
-    "cons": {
-        "scrape-url": [(60, 60), (3000, 24 * 3600 * 30)],
-        "bulk": [(60, 60), (3000, 24 * 3600 * 30)],
-        "fast": [(120, 60), (30_000, 24 * 3600 * 30)],
-    },
-    "stup": {
-        "scrape-url": [(60, 60), (30_000, 24 * 3600 * 30)],
-        "bulk": [(60, 60), (30_000, 24 * 3600 * 30)],
-        "fast": [(360, 60), (300_000, 24 * 3600 * 30)],
-    },
-    # This plan is used for testing in the package
-    "busn": {
-        "scrape-url": [(60, 60), (300_000, 24 * 3600 * 30)],
-        "bulk": [(60, 60), (300_000, 24 * 3600 * 30)],
-        "fast": [(360, 60), (3_000_000, 24 * 3600 * 30)],
-    },
-}
-
 
 def _rate_limited(endpoint):
     """
@@ -133,7 +62,7 @@ class RateLimiter:
 
         # Build our bucket
         bucket = InMemoryBucket([Rate(max_calls, period_ms)])
-        self._limiter = Limiter(bucket)
+        self._limiter = Limiter(bucket, max_delay=1000)
 
     def acquire(self) -> None:
         """
@@ -177,7 +106,7 @@ class RateLimiter:
                 # Ensure at least a small sleep if rounding to zero
                 time.sleep(wait_s)
 
-    def try_acquire(self) -> bool:
+    def try_acquire(self, name: str = None) -> bool:
         """
         Attempt to acquire a slot without blocking.
 
@@ -196,8 +125,11 @@ class RateLimiter:
         >>> rl.try_acquire()
         False
         """
+        key = name if name else self._GLOBAL_KEY
+
         try:
-            self._limiter.try_acquire(self._GLOBAL_KEY)
+            self._limiter.try_acquire(key)
             return True
-        except BucketFullException:
-            return False
+        except (BucketFullException, LimiterDelayException):
+            # Return False instead of crashing when the limit is hit
+            return False

{nosible-0.3.6 → nosible-0.3.10/src/nosible.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nosible
-Version: 0.3.6
+Version: 0.3.10
 Summary: Python client for the NOSIBLE Search API
 Home-page: https://github.com/NosibleAI/nosible-py
 Author: Stuart Reid, Matthew Dicks, Richard Taylor, Gareth Warburton
@@ -31,7 +31,7 @@ Requires-Dist: polars
 Requires-Dist: duckdb
 Requires-Dist: openai
 Requires-Dist: tantivy
-Requires-Dist: pyrate-limiter
+Requires-Dist: pyrate-limiter<4
 Requires-Dist: tenacity
 Requires-Dist: cryptography
 Requires-Dist: pyarrow

{nosible-0.3.6 → nosible-0.3.10}/src/nosible.egg-info/requires.txt

@@ -2,7 +2,7 @@ polars
 duckdb
 openai
 tantivy
-pyrate-limiter
+pyrate-limiter<4
 tenacity
 cryptography
 pyarrow

{nosible-0.3.6 → nosible-0.3.10}/tests/test_01_nosible.py

@@ -1,12 +1,9 @@
-import json
 import pytest
-import os
-import time
 import re
 
-import polars as pl
+import pytest
 
-from nosible import Nosible, Result, ResultSet, Search, Snippet, SnippetSet
+from nosible import Nosible, ResultSet, Search, SnippetSet
 from nosible.classes.search_set import SearchSet
 from nosible.classes.web_page import WebPageData
 
@@ -75,11 +72,6 @@ def test_close_idempotent():
     nos.close()
 
 
-def test_invalid_api_key():
-    with pytest.raises(ValueError):
-        Nosible(nosible_api_key="test+|xyz")
-
-
 def test_llm_key_required_for_expansions():
     nos = Nosible(llm_api_key=None)
     nos.llm_api_key = None
@@ -92,9 +84,6 @@ def test_validate_sql():
     assert not Nosible()._validate_sql(sql="SELECT * FROM missing_table")
 
 
-# —— Your additional tests —— #
-
-
 def test_search_minimal(search_data):
     # from your snippet: isinstance(search_data, ResultSet)
     assert isinstance(search_data, ResultSet)

{nosible-0.3.6 → nosible-0.3.10}/tests/test_02_results.py

@@ -1,5 +1,5 @@
 import pytest
-from polars.dependencies import pandas as pd
+import pandas as pd
 from nosible import Result, ResultSet
 
 
@@ -84,8 +84,6 @@ def test_resultset_to_dict(search_data):
         assert "published" in res
         assert "similarity" in res
         assert res["url_hash"] == key
-    # results_copy_from_dict = ResultSet.from_dict(results_dict)
-    # assert results == results_copy_from_dict
 
 
 # to_dicts
@@ -149,3 +147,32 @@ def test_resultset_getitem(search_data):
         _ = search_data[len(search_data)]  # Out of range index
     with pytest.raises(TypeError):
         _ = search_data["invalid"]  # Invalid type for index
+
+
+def test_similar_excludes_current_document():
+    """
+    Test that the similar method properly excludes the current document from search results.
+
+    This test creates a Nosible client, performs a fast search, takes the first result,
+    and verifies that calling similar() on that result excludes it from the returned results.
+    """
+    from nosible import Nosible
+
+    # Create a Nosible client (similar to test_01_nosible.py)
+    with Nosible(concurrency=1) as nos:
+        # Perform a search to get some results
+        search_results = nos.fast_search(question="Hedge funds seek to expand into private credit", n_results=10)
+
+        # Get the first result
+        first_result = search_results[0]
+
+        # Call similar() on the first result
+        similar_results = first_result.similar(client=nos, n_results=10)
+
+        # Verify that the first result is NOT in the similar results
+        # We check by comparing URL hashes
+        similar_hashes = [r.url_hash for r in similar_results if r.url_hash]
+        assert first_result.url_hash not in similar_hashes, f"Original result URL hash {first_result.url_hash} should not be in similar results"
+
+        # Also verify that similar results were actually returned (should be non-empty)
+        assert len(similar_results) >= 0, "Similar results should be returned (may be empty if no similar docs found)"

{nosible-0.3.6 → nosible-0.3.10}/tests/test_04_snippets.py

@@ -1,5 +1,4 @@
-from nosible import Snippet, SnippetSet, WebPageData
-import pytest
+from nosible import Snippet, SnippetSet
 
 
 def test_snippet_initialization(snippets_data):