nosible 0.3.9__py3-none-any.whl → 0.3.10__py3-none-any.whl

nosible/nosible_client.py

@@ -29,7 +29,7 @@ from nosible.classes.search_set import SearchSet
 from nosible.classes.snippet_set import SnippetSet
 from nosible.classes.web_page import WebPageData
 from nosible.utils.json_tools import json_loads
-from nosible.utils.rate_limiter import PLAN_RATE_LIMITS, RateLimiter, _rate_limited
+from nosible.utils.rate_limiter import RateLimiter, _rate_limited
 
 # Set up a module‐level logger.
 logger = logging.getLogger(__name__)
@@ -202,11 +202,6 @@ class Nosible:
         logging.getLogger("httpx").setLevel(logging.WARNING)
         logging.getLogger("httpcore").setLevel(logging.WARNING)
 
-        self._limiters = {
-            endpoint: [RateLimiter(calls, period) for calls, period in buckets]
-            for endpoint, buckets in PLAN_RATE_LIMITS[self._get_user_plan()].items()
-        }
-
         # Define retry decorator
         self._post = retry(
             reraise=True,
@@ -230,7 +225,34 @@ class Nosible:
         self._executor = ThreadPoolExecutor(max_workers=self.concurrency)
 
         # Headers
-        self.headers = {"Accept-Encoding": "gzip", "Content-Type": "application/json", "api-key": self.nosible_api_key}
+        self.headers = {
+            "Accept-Encoding": "gzip",
+            "Content-Type": "application/json",
+            "api-key": self.nosible_api_key
+        }
+
+        # Wrap _get_limits with retry.
+        self._get_limits = retry(
+            reraise=True,
+            stop=stop_after_attempt(self.retries) | stop_after_delay(self.timeout),
+            wait=wait_exponential(multiplier=1, min=1, max=20),
+            retry=retry_if_exception_type(httpx.RequestError),
+            before_sleep=before_sleep_log(self.logger, logging.WARNING),
+        )(self._get_limits)
+
+        raw_limits = self._get_limits()
+
+        # Map API query_type -> your decorator endpoint keys
+        mapped_limits = {
+            "fast": raw_limits.get("fast", []),
+            "bulk": raw_limits.get("slow", []),
+            "scrape-url": raw_limits.get("visit", []),
+        }
+
+        self._limiters = {
+            endpoint: [RateLimiter(calls, period) for calls, period in buckets]
+            for endpoint, buckets in mapped_limits.items()
+        }
 
         # Filters
         self.publish_start = publish_start
@@ -1602,7 +1624,6 @@ class Nosible:
 
         return filtered
 
-
     def close(self):
         """
         Close the Nosible client, shutting down the HTTP session
@@ -1702,41 +1723,52 @@ class Nosible:
 
         return response
 
-    def _get_user_plan(self) -> str:
+    def _get_limits(self) -> dict[str, list[tuple[int, float]]]:
+        """
+        TODO
         """
-        Determine the user's subscription plan from the API key.
+        url = "https://www.nosible.ai/search/v2/limits"
+        resp = self._session.get(
+            url=url,
+            headers=self.headers,
+            timeout=self.timeout,
+            follow_redirects=True,
+        )
 
-        The `nosible_api_key` is expected to start with a plan prefix followed by
-        a pipe (`|`) and any additional data. This method splits on the first
-        pipe character, validates the prefix against supported plans, and returns it.
+        if resp.status_code == 401:
+            raise ValueError("Your API key is not valid.")
+        if resp.status_code == 429:
+            raise ValueError("You have hit your rate limit.")
+        if resp.status_code == 409:
+            raise ValueError("Too many concurrent searches.")
+        if resp.status_code == 502:
+            raise ValueError("NOSIBLE is currently restarting.")
+        if resp.status_code == 504:
+            raise ValueError("NOSIBLE is currently overloaded.")
 
-        Returns
-        -------
-        str
-            The plan you are currently on.
+        resp.raise_for_status()
 
-        Raises
-        ------
-        ValueError
-            If the extracted prefix is not one of the recognized plan names.
+        try:
+            data = resp.json()
+        except Exception as e:
+            raise ValueError("Invalid JSON response from /limits") from e
 
-        Examples
-        --------
-        >>> nos = Nosible(nosible_api_key="test+|xyz")  # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
-        Traceback (most recent call last):
-        ...
-        ValueError: Your API key is not valid: test+ is not a valid plan prefix.
-        """
-        # Split off anything after the first '|'
-        prefix = (self.nosible_api_key or "").split("|", 1)[0]
+        limits_list = data.get("limits")
+        if not isinstance(limits_list, list):
+            raise ValueError(f"Invalid /limits response shape: {data!r}")
+
+        grouped: dict[str, list[tuple[int, float]]] = {}
+        for item in limits_list:
+            query_type = item.get("query_type")
+            duration = item.get("duration_seconds")
+            limit = item.get("limit")
 
-        # Map prefixes -> plan names
-        plans = {"test", "self", "basic", "pro", "pro+", "bus", "bus+", "ent", "chat", "cons", "stup", "busn", "prod"}
+            if query_type is None or duration is None or limit is None:
+                raise ValueError(f"Invalid limit entry: {item!r}")
 
-        if prefix not in plans:
-            raise ValueError(f"Your API key is not valid: {prefix} is not a valid plan prefix.")
+            grouped.setdefault(str(query_type), []).append((int(limit), float(duration)))
 
-        return prefix
+        return grouped
 
     def _generate_expansions(self, question: Union[str, Search]) -> list:
         """

nosible/utils/rate_limiter.py

@@ -8,77 +8,6 @@ from pyrate_limiter.exceptions import BucketFullException, LimiterDelayException
 
 log = logging.getLogger(__name__)
 
-PLAN_RATE_LIMITS = {
-    "test": {
-        # Per minute limit, then per month.
-        "scrape-url": [(60, 60), (300, 24 * 3600 * 30)],
-        "bulk": [(60, 60), (300, 24 * 3600 * 30)],
-        "fast": [(60, 60), (3000, 24 * 3600 * 30)],
-    },
-    "basic": {
-        "scrape-url": [(60, 60), (1400, 24 * 3600 * 30)],
-        "bulk": [(60, 60), (1400, 24 * 3600 * 30)],
-        "fast": [(60, 60), (14_000, 24 * 3600 * 30)],
-    },
-    "pro": {
-        "scrape-url": [(60, 60), (6700, 24 * 3600 * 30)],
-        "bulk": [(60, 60), (6700, 24 * 3600 * 30)],
-        "fast": [(60, 60), (67_000, 24 * 3600 * 30)],
-    },
-    "pro+": {
-        "scrape-url": [(60, 60), (32_000, 24 * 3600 * 30)],
-        "bulk": [(60, 60), (32_000, 24 * 3600 * 30)],
-        "fast": [(60, 60), (320_000, 24 * 3600 * 30)],
-    },
-    "bus": {
-        "scrape-url": [(60, 60), (200_000, 24 * 3600 * 30)],
-        "bulk": [(60, 60), (200_000, 24 * 3600 * 30)],
-        "fast": [(60, 60), (2_000_000, 24 * 3600 * 30)],
-    },
-    "bus+": {
-        "scrape-url": [(60, 60), (500_000, 24 * 3600 * 30)],
-        "bulk": [(60, 60), (500_000, 24 * 3600 * 30)],
-        "fast": [(120, 60), (5_000_000, 24 * 3600 * 30)],
-    },
-    "ent": {
-        "scrape-url": [(60, 60), (1_500_000, 24 * 3600 * 30)],
-        "bulk": [(60, 60), (1_500_000, 24 * 3600 * 30)],
-        "fast": [(360, 60), (15_000_000, 24 * 3600 * 30)],
-    },
-    "prod": {
-        "scrape-url": [(60, 60), (1_500_000, 24 * 3600 * 30)],
-        "bulk": [(60, 60), (1_500_000, 24 * 3600 * 30)],
-        "fast": [(360, 60), (15_000_000, 24 * 3600 * 30)],
-    },
-    # This plan is used for testing in the package
-    "chat": {
-        "scrape-url": [(60, 60), (1_500_000, 24 * 3600 * 30)],
-        "bulk": [(60, 60), (1_500_000, 24 * 3600 * 30)],
-        "fast": [(360, 60), (15_000_000, 24 * 3600 * 30)],
-    },
-    "self": {
-        "scrape-url": [(6000, 60), (1_500_000, 24 * 3600 * 30)],
-        "bulk": [(6000, 60), (1_500_000, 24 * 3600 * 30)],
-        "fast": [(36_000, 60), (15_000_000, 24 * 3600 * 30)],
-    },
-    "cons": {
-        "scrape-url": [(60, 60), (3000, 24 * 3600 * 30)],
-        "bulk": [(60, 60), (3000, 24 * 3600 * 30)],
-        "fast": [(120, 60), (30_000, 24 * 3600 * 30)],
-    },
-    "stup": {
-        "scrape-url": [(60, 60), (30_000, 24 * 3600 * 30)],
-        "bulk": [(60, 60), (30_000, 24 * 3600 * 30)],
-        "fast": [(360, 60), (300_000, 24 * 3600 * 30)],
-    },
-    # This plan is used for testing in the package
-    "busn": {
-        "scrape-url": [(60, 60), (300_000, 24 * 3600 * 30)],
-        "bulk": [(60, 60), (300_000, 24 * 3600 * 30)],
-        "fast": [(360, 60), (3_000_000, 24 * 3600 * 30)],
-    },
-}
-
 
 def _rate_limited(endpoint):
     """

{nosible-0.3.9.dist-info → nosible-0.3.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nosible
-Version: 0.3.9
+Version: 0.3.10
 Summary: Python client for the NOSIBLE Search API
 Home-page: https://github.com/NosibleAI/nosible-py
 Author: Stuart Reid, Matthew Dicks, Richard Taylor, Gareth Warburton
@@ -31,7 +31,7 @@ Requires-Dist: polars
 Requires-Dist: duckdb
 Requires-Dist: openai
 Requires-Dist: tantivy
-Requires-Dist: pyrate-limiter
+Requires-Dist: pyrate-limiter<4
 Requires-Dist: tenacity
 Requires-Dist: cryptography
 Requires-Dist: pyarrow

{nosible-0.3.9.dist-info → nosible-0.3.10.dist-info}/RECORD

@@ -1,5 +1,5 @@
 nosible/__init__.py,sha256=11QmG9Wjprp_zB0VnPxGjqKwHmaoB0hoT8AGO6cGVMM,1426
-nosible/nosible_client.py,sha256=tTnDts1XaxaqGRW4ymcwYsZlbzg2i_1Qw9GfD3PCRmA,88572
+nosible/nosible_client.py,sha256=nq9WKGVCyarG5-3vXZOT67vLfGq7rlSEOBbqfLZpEE0,89644
 nosible/classes/result.py,sha256=DvUUt4hiZ8PJBGTIwkq5GFB4ors_2L9dApNlawSLesA,22978
 nosible/classes/result_set.py,sha256=9xuACJVp5n3pKECBjhkzNMb7PAQd4P1HmtkZH3KFAyo,52911
 nosible/classes/search.py,sha256=VrpuAsS4pxcirR6l6WStJN0gPIfahZ_9Cx3YdNKczSw,13860
@@ -8,9 +8,9 @@ nosible/classes/snippet.py,sha256=K0o4aQbbvsnNzgnhZIu7RXLASkVO-R3iBIY7sCpNpLw,47
 nosible/classes/snippet_set.py,sha256=0jPMDhJNCO02WhvY1QR1HedvADvBxRcN6x3FItEgSiI,5099
 nosible/classes/web_page.py,sha256=gAOBnqaK59f5erdRERxhvKmcwRVSXvhwJwyHIjPyeek,5732
 nosible/utils/json_tools.py,sha256=PcSMjcLEhbA626jAIn0SuD_1-4QDduapZUenTSt3N2E,4569
-nosible/utils/rate_limiter.py,sha256=X1-X5ykU9NKpp1vZJQj1oXEEBUQvur7PSG_o4Yidp4k,6657
-nosible-0.3.9.dist-info/licenses/LICENSE,sha256=8ifsV4DrsiKi8KVBFy8SBb3KXPXhofE3pYq07q1TSCQ,1117
-nosible-0.3.9.dist-info/METADATA,sha256=8O3piycx24IomgyZhErKfDHyiH3DeY91iHOPC4l0GcQ,6032
-nosible-0.3.9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nosible-0.3.9.dist-info/top_level.txt,sha256=mOconHuKcNJ1jTAj3DapQP_xB8YOmjTMyHg5txKH3uA,8
-nosible-0.3.9.dist-info/RECORD,,
+nosible/utils/rate_limiter.py,sha256=Io7OCU6MmGoPswYj6PivtSNkFdzLaeam3tf29F5XeUQ,3980
+nosible-0.3.10.dist-info/licenses/LICENSE,sha256=8ifsV4DrsiKi8KVBFy8SBb3KXPXhofE3pYq07q1TSCQ,1117
+nosible-0.3.10.dist-info/METADATA,sha256=oWvzqQvtmgzhcaxIQ0APzsqoE_PSuSzVT_RWTYk5tQ8,6035
+nosible-0.3.10.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+nosible-0.3.10.dist-info/top_level.txt,sha256=mOconHuKcNJ1jTAj3DapQP_xB8YOmjTMyHg5txKH3uA,8
+nosible-0.3.10.dist-info/RECORD,,

{nosible-0.3.9.dist-info → nosible-0.3.10.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any