python-documentcloud 4.6.0__tar.gz → 4.7.0__tar.gz

{python_documentcloud-4.6.0/python_documentcloud.egg-info → python_documentcloud-4.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-documentcloud
-Version: 4.6.0
+Version: 4.7.0
 Summary: A simple Python wrapper for the DocumentCloud API
 Home-page: https://github.com/muckrock/python-documentcloud
 Author: Mitchell Kotler
@@ -27,6 +27,7 @@ Requires-Dist: urllib3
 Requires-Dist: pyyaml
 Requires-Dist: fastjsonschema
 Requires-Dist: python-squarelet
+Requires-Dist: token-bucket
 Provides-Extra: dev
 Requires-Dist: black; extra == "dev"
 Requires-Dist: coverage; extra == "dev"
@@ -37,6 +38,7 @@ Requires-Dist: twine; extra == "dev"
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: pytest-mock; extra == "test"
+Requires-Dist: pytest-xdist; extra == "test"
 Requires-Dist: pytest-recording; extra == "test"
 Requires-Dist: vcrpy; extra == "test"
 Dynamic: author

python_documentcloud-4.7.0/documentcloud/client.py

@@ -0,0 +1,100 @@
+# Standard Library
+import logging
+import time
+
+# Third Party
+import token_bucket
+from squarelet import SquareletClient
+
+# Local
+from .documents import DocumentClient
+from .organizations import OrganizationClient
+from .projects import ProjectClient
+from .users import UserClient
+
+logger = logging.getLogger("documentcloud")
+
+# Per-endpoint rate limits applied on top of the global squarelet limit.
+# Format: (method, url_pattern, rate_per_second, capacity)
+#
+# Endpoint              Rate        Burst   Notes
+# --------              ----        -----   -----
+# GET  documents/search  15/min      50
+# POST documents/        12/min      100     25 docs/bulk call = up to 300 docs/min
+# PUT  documents/        12/min      100     25 docs/bulk call = up to 300 docs/min
+# GET  files/            15/min      100     PDFs, full text, and other private assets
+ENDPOINT_RATE_LIMITS = [
+    ("GET", "documents/search", 15 / 60, 50),
+    ("POST", "documents/", 12 / 60, 100),
+    ("PUT", "documents/", 12 / 60, 100),
+    ("GET", "files/", 15 / 60, 100),
+]
+
+
+class DocumentCloud(SquareletClient):
+    """
+    The public interface for the DocumentCloud API, now integrated with SquareletClient
+    """
+
+    def __init__(
+        self,
+        username=None,
+        password=None,
+        base_uri="https://api.www.documentcloud.org/api/",
+        auth_uri="https://accounts.muckrock.com/api/",
+        timeout=20,
+        loglevel=None,
+        rate_limit=True,
+        rate_limit_sleep=True,
+    ):
+        # Initialize SquareletClient for authentication and request handling
+        super().__init__(
+            base_uri=base_uri,
+            username=username,
+            password=password,
+            auth_uri=auth_uri,
+            timeout=timeout,
+            rate_limit=rate_limit,
+            rate_limit_sleep=rate_limit_sleep,
+        )
+
+        # Set up logging
+        if loglevel:
+            logging.basicConfig(
+                level=loglevel,
+                format="%(asctime)s %(levelname)-8s %(name)-25s %(message)s",
+            )
+        else:
+            logger.addHandler(logging.NullHandler())
+
+        # Build per-endpoint token bucket rate limiters
+        storage = token_bucket.MemoryStorage()
+        self._endpoint_limiters = [
+            (
+                pattern_method,
+                pattern,
+                token_bucket.Limiter(rate=rate, capacity=capacity, storage=storage),
+                f"{pattern_method}:{pattern}",
+            )
+            for pattern_method, pattern, rate, capacity in ENDPOINT_RATE_LIMITS
+        ]
+
+        # Initialize the sub-clients using SquareletClient
+        self.documents = DocumentClient(self)
+        self.projects = ProjectClient(self)
+        self.users = UserClient(self)
+        self.organizations = OrganizationClient(self)
+
+    def request(self, method, url, raise_error=True, **kwargs):
+        for pattern_method, pattern, limiter, bucket_key in self._endpoint_limiters:
+            if pattern_method.upper() == method.upper() and pattern in url:
+                if not limiter.consume(bucket_key):
+                    logger.warning(
+                        "Rate limit reached for %s %s, throttling...",
+                        method.upper(),
+                        pattern,
+                    )
+                    while not limiter.consume(bucket_key):
+                        time.sleep(0.1)
+                return super().request(method, url, raise_error=raise_error, **kwargs)
+        return super().request(method, url, raise_error=raise_error, **kwargs)

{python_documentcloud-4.6.0 → python_documentcloud-4.7.0}/documentcloud/documents.py

@@ -7,11 +7,13 @@ import datetime
 import logging
 import os
 import re
+import time
 import warnings
 from functools import partial
 from urllib.parse import urlparse
 
 # Third Party
+import token_bucket
 from requests.exceptions import RequestException
 
 # Local
@@ -28,6 +30,8 @@ logger = logging.getLogger("documentcloud")
 
 IMAGE_SIZES = ["thumbnail", "small", "normal", "large", "xlarge"]
 
+DEFAULT_USER_AGENT = "python-documentcloud"
+
 
 class Document(BaseAPIObject):
     """A single DocumentCloud document"""
@@ -164,12 +168,17 @@ class Document(BaseAPIObject):
 
         if base_netloc == url_netloc:
             # if the url host is the same as the base api host,
-            # sent the request with the client in order to include
+            # send the request with the client in order to include
             # authentication credentials
             response = self._client.get(url, full_url=True)
         else:
-            response = requests_retry_session().get(
-                url, headers={"User-Agent": "python-documentcloud2"}
+            response = self._client.documents.asset_get(
+                url,
+                headers={
+                    "User-Agent": self._client.session.headers.get(
+                        "User-Agent", DEFAULT_USER_AGENT
+                    )
+                },
             )
         if fmt == "text":
             return response.content.decode("utf8")
@@ -246,6 +255,26 @@ class DocumentClient(BaseAPIClient):
     api_path = "documents"
     resource = Document
 
+    def __init__(self, client):
+        super().__init__(client)
+        # Rate limit for public document asset fetches (S3-hosted).
+        # Private document assets go through the API client and are limited there.
+        # Token bucket: burst of 100, sustained at 15/min (0.25/sec).
+        storage = token_bucket.MemoryStorage()
+        self._asset_limiter = token_bucket.Limiter(
+            rate=15 / 60,
+            capacity=100,
+            storage=storage,
+        )
+        self._asset_session = requests_retry_session()
+
+    def asset_get(self, url, **kwargs):
+        if not self._asset_limiter.consume("asset"):
+            logger.warning("Rate limit reached for asset fetch, throttling...")
+            while not self._asset_limiter.consume("asset"):
+                time.sleep(0.1)
+        return self._asset_session.get(url, **kwargs)
+
     def search(self, query, **params):
         """Return documents matching a search query"""
 

{python_documentcloud-4.6.0 → python_documentcloud-4.7.0/python_documentcloud.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-documentcloud
-Version: 4.6.0
+Version: 4.7.0
 Summary: A simple Python wrapper for the DocumentCloud API
 Home-page: https://github.com/muckrock/python-documentcloud
 Author: Mitchell Kotler
@@ -27,6 +27,7 @@ Requires-Dist: urllib3
 Requires-Dist: pyyaml
 Requires-Dist: fastjsonschema
 Requires-Dist: python-squarelet
+Requires-Dist: token-bucket
 Provides-Extra: dev
 Requires-Dist: black; extra == "dev"
 Requires-Dist: coverage; extra == "dev"
@@ -37,6 +38,7 @@ Requires-Dist: twine; extra == "dev"
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: pytest-mock; extra == "test"
+Requires-Dist: pytest-xdist; extra == "test"
 Requires-Dist: pytest-recording; extra == "test"
 Requires-Dist: vcrpy; extra == "test"
 Dynamic: author

{python_documentcloud-4.6.0 → python_documentcloud-4.7.0}/python_documentcloud.egg-info/requires.txt

@@ -7,6 +7,7 @@ urllib3
 pyyaml
 fastjsonschema
 python-squarelet
+token-bucket
 
 [dev]
 black
@@ -19,5 +20,6 @@ twine
 [test]
 pytest
 pytest-mock
+pytest-xdist
 pytest-recording
 vcrpy

{python_documentcloud-4.6.0 → python_documentcloud-4.7.0}/setup.py

@@ -7,7 +7,7 @@ with open("README.md", "r") as fh:
 
 setup(
     name="python-documentcloud",
-    version="4.6.0",
+    version="4.7.0",
     description="A simple Python wrapper for the DocumentCloud API",
     author="Mitchell Kotler",
     author_email="mitch@muckrock.com",
@@ -27,6 +27,7 @@ setup(
         "pyyaml",
         "fastjsonschema",
         "python-squarelet",
+        "token-bucket",
     ),
     extras_require={
         "dev": [
@@ -40,6 +41,7 @@ setup(
         "test": [
             "pytest",
             "pytest-mock",
+            "pytest-xdist",
             "pytest-recording",
             "vcrpy",
         ],

{python_documentcloud-4.6.0 → python_documentcloud-4.7.0}/tests/test_client.py

@@ -9,6 +9,7 @@ import pytest
 import ratelimit
 
 # DocumentCloud
+from documentcloud import DocumentCloud
 from documentcloud.constants import RATE_LIMIT
 from documentcloud.exceptions import APIError, CredentialsFailedError
 
@@ -111,3 +112,77 @@ def test_expired_refresh_token(short_client, record_mode):
     assert short_client.users.get("me")
     # check the refresh token was updated
     assert old_refresh_token != short_client.refresh_token
+
+
+def test_endpoint_rate_limit_burst_exhaustion():
+    """Token bucket should block after burst capacity is exhausted"""
+    client = DocumentCloud()
+    # Exhaust the search burst (capacity=50)
+    _pattern_method, _pattern, limiter, bucket_key = client._endpoint_limiters[0]
+    for _ in range(50):
+        limiter.consume(bucket_key)
+    assert not limiter.consume(bucket_key)
+
+
+def test_endpoint_rate_limit_method_specificity():
+    """GET and POST to documents/ should use different limiters"""
+    client = DocumentCloud()
+    limiters = {(pm, p): lim for pm, p, lim, _ in client._endpoint_limiters}
+    assert limiters[("GET", "files/")] is not limiters[("POST", "documents/")]
+
+
+def test_endpoint_rate_limit_pattern_ordering():
+    """documents/search should match before documents/"""
+    client = DocumentCloud()
+    url = "documents/search/"
+    matched = next(
+        p for pm, p, _, _ in client._endpoint_limiters if pm == "GET" and p in url
+    )
+    assert matched == "documents/search"
+
+
+def test_asset_rate_limit_burst_exhaustion():
+    """Asset token bucket should block after burst capacity is exhausted"""
+    client = DocumentCloud()
+    limiter = client.documents._asset_limiter
+    for _ in range(100):
+        limiter.consume("asset")
+    assert not limiter.consume("asset")
+
+
+def test_asset_rate_limit_refills():
+    """Asset token bucket should refill over time"""
+    client = DocumentCloud()
+    limiter = client.documents._asset_limiter
+    for _ in range(100):
+        limiter.consume("asset")
+    assert not limiter.consume("asset")
+    time.sleep(5)
+    assert limiter.consume("asset")
+
+
+def test_endpoint_rate_limit_buckets_are_independent():
+    """Exhausting one endpoint's bucket should not affect another"""
+    client = DocumentCloud()
+    limiters = {(pm, p): (lim, bk) for pm, p, lim, bk in client._endpoint_limiters}
+    search_limiter, search_key = limiters[("GET", "documents/search")]
+    files_limiter, files_key = limiters[("GET", "files/")]
+
+    # Exhaust search bucket
+    for _ in range(50):
+        search_limiter.consume(search_key)
+    assert not search_limiter.consume(search_key)
+
+    # Files bucket should still have tokens
+    assert files_limiter.consume(files_key)
+
+
+def test_endpoint_rate_limit_no_match_for_unrecognized_url():
+    """Unrecognized URLs should not match any endpoint limiter"""
+    client = DocumentCloud()
+    url = "users/me/"
+    matched = next(
+        (p for pm, p, _, _ in client._endpoint_limiters if p in url),
+        None,
+    )
+    assert matched is None

python_documentcloud-4.6.0/documentcloud/client.py

@@ -1,58 +0,0 @@
-# Import SquareletClient from python-squarelet
-# Standard Library
-import logging
-
-# Third Party
-from squarelet import SquareletClient
-
-# Local
-# Local Imports
-from .documents import DocumentClient
-from .organizations import OrganizationClient
-from .projects import ProjectClient
-from .users import UserClient
-
-logger = logging.getLogger("documentcloud")
-
-
-class DocumentCloud(SquareletClient):
-    """
-    The public interface for the DocumentCloud API, now integrated with SquareletClient
-    """
-
-    def __init__(
-        self,
-        username=None,
-        password=None,
-        base_uri="https://api.www.documentcloud.org/api/",
-        auth_uri="https://accounts.muckrock.com/api/",
-        timeout=20,
-        loglevel=None,
-        rate_limit=True,
-        rate_limit_sleep=True,
-    ):
-        # Initialize SquareletClient for authentication and request handling
-        super().__init__(
-            base_uri=base_uri,
-            username=username,
-            password=password,
-            auth_uri=auth_uri,
-            timeout=timeout,
-            rate_limit=rate_limit,
-            rate_limit_sleep=rate_limit_sleep,
-        )
-
-        # Set up logging
-        if loglevel:
-            logging.basicConfig(
-                level=loglevel,
-                format="%(asctime)s %(levelname)-8s %(name)-25s %(message)s",
-            )
-        else:
-            logger.addHandler(logging.NullHandler())
-
-        # Initialize the sub-clients using SquareletClient
-        self.documents = DocumentClient(self)
-        self.projects = ProjectClient(self)
-        self.users = UserClient(self)
-        self.organizations = OrganizationClient(self)