python-documentcloud 4.6.0__py2.py3-none-any.whl → 4.7.0__py2.py3-none-any.whl

documentcloud/client.py

@@ -1,12 +1,12 @@
-# Import SquareletClient from python-squarelet
 # Standard Library
 import logging
+import time
 
 # Third Party
+import token_bucket
 from squarelet import SquareletClient
 
 # Local
-# Local Imports
 from .documents import DocumentClient
 from .organizations import OrganizationClient
 from .projects import ProjectClient
@@ -14,6 +14,22 @@ from .users import UserClient
 
 logger = logging.getLogger("documentcloud")
 
+# Per-endpoint rate limits applied on top of the global squarelet limit.
+# Format: (method, url_pattern, rate_per_second, capacity)
+#
+# Endpoint              Rate        Burst   Notes
+# --------              ----        -----   -----
+# GET  documents/search  15/min      50
+# POST documents/        12/min      100     25 docs/bulk call = up to 300 docs/min
+# PUT  documents/        12/min      100     25 docs/bulk call = up to 300 docs/min
+# GET  files/            15/min      100     PDFs, full text, and other private assets
+ENDPOINT_RATE_LIMITS = [
+    ("GET", "documents/search", 15 / 60, 50),
+    ("POST", "documents/", 12 / 60, 100),
+    ("PUT", "documents/", 12 / 60, 100),
+    ("GET", "files/", 15 / 60, 100),
+]
+
 
 class DocumentCloud(SquareletClient):
     """
@@ -51,8 +67,34 @@ class DocumentCloud(SquareletClient):
         else:
             logger.addHandler(logging.NullHandler())
 
+        # Build per-endpoint token bucket rate limiters
+        storage = token_bucket.MemoryStorage()
+        self._endpoint_limiters = [
+            (
+                pattern_method,
+                pattern,
+                token_bucket.Limiter(rate=rate, capacity=capacity, storage=storage),
+                f"{pattern_method}:{pattern}",
+            )
+            for pattern_method, pattern, rate, capacity in ENDPOINT_RATE_LIMITS
+        ]
+
         # Initialize the sub-clients using SquareletClient
         self.documents = DocumentClient(self)
         self.projects = ProjectClient(self)
         self.users = UserClient(self)
         self.organizations = OrganizationClient(self)
+
+    def request(self, method, url, raise_error=True, **kwargs):
+        for pattern_method, pattern, limiter, bucket_key in self._endpoint_limiters:
+            if pattern_method.upper() == method.upper() and pattern in url:
+                if not limiter.consume(bucket_key):
+                    logger.warning(
+                        "Rate limit reached for %s %s, throttling...",
+                        method.upper(),
+                        pattern,
+                    )
+                    while not limiter.consume(bucket_key):
+                        time.sleep(0.1)
+                return super().request(method, url, raise_error=raise_error, **kwargs)
+        return super().request(method, url, raise_error=raise_error, **kwargs)

documentcloud/documents.py

@@ -7,11 +7,13 @@ import datetime
 import logging
 import os
 import re
+import time
 import warnings
 from functools import partial
 from urllib.parse import urlparse
 
 # Third Party
+import token_bucket
 from requests.exceptions import RequestException
 
 # Local
@@ -28,6 +30,8 @@ logger = logging.getLogger("documentcloud")
 
 IMAGE_SIZES = ["thumbnail", "small", "normal", "large", "xlarge"]
 
+DEFAULT_USER_AGENT = "python-documentcloud"
+
 
 class Document(BaseAPIObject):
     """A single DocumentCloud document"""
@@ -164,12 +168,17 @@ class Document(BaseAPIObject):
 
         if base_netloc == url_netloc:
             # if the url host is the same as the base api host,
-            # sent the request with the client in order to include
+            # send the request with the client in order to include
             # authentication credentials
             response = self._client.get(url, full_url=True)
         else:
-            response = requests_retry_session().get(
-                url, headers={"User-Agent": "python-documentcloud2"}
+            response = self._client.documents.asset_get(
+                url,
+                headers={
+                    "User-Agent": self._client.session.headers.get(
+                        "User-Agent", DEFAULT_USER_AGENT
+                    )
+                },
             )
         if fmt == "text":
             return response.content.decode("utf8")
@@ -246,6 +255,26 @@ class DocumentClient(BaseAPIClient):
     api_path = "documents"
     resource = Document
 
+    def __init__(self, client):
+        super().__init__(client)
+        # Rate limit for public document asset fetches (S3-hosted).
+        # Private document assets go through the API client and are limited there.
+        # Token bucket: burst of 100, sustained at 15/min (0.25/sec).
+        storage = token_bucket.MemoryStorage()
+        self._asset_limiter = token_bucket.Limiter(
+            rate=15 / 60,
+            capacity=100,
+            storage=storage,
+        )
+        self._asset_session = requests_retry_session()
+
+    def asset_get(self, url, **kwargs):
+        if not self._asset_limiter.consume("asset"):
+            logger.warning("Rate limit reached for asset fetch, throttling...")
+            while not self._asset_limiter.consume("asset"):
+                time.sleep(0.1)
+        return self._asset_session.get(url, **kwargs)
+
     def search(self, query, **params):
         """Return documents matching a search query"""
 

{python_documentcloud-4.6.0.dist-info → python_documentcloud-4.7.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-documentcloud
-Version: 4.6.0
+Version: 4.7.0
 Summary: A simple Python wrapper for the DocumentCloud API
 Home-page: https://github.com/muckrock/python-documentcloud
 Author: Mitchell Kotler
@@ -27,6 +27,7 @@ Requires-Dist: urllib3
 Requires-Dist: pyyaml
 Requires-Dist: fastjsonschema
 Requires-Dist: python-squarelet
+Requires-Dist: token-bucket
 Provides-Extra: dev
 Requires-Dist: black; extra == "dev"
 Requires-Dist: coverage; extra == "dev"
@@ -37,6 +38,7 @@ Requires-Dist: twine; extra == "dev"
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: pytest-mock; extra == "test"
+Requires-Dist: pytest-xdist; extra == "test"
 Requires-Dist: pytest-recording; extra == "test"
 Requires-Dist: vcrpy; extra == "test"
 Dynamic: author

{python_documentcloud-4.6.0.dist-info → python_documentcloud-4.7.0.dist-info}/RECORD

@@ -2,17 +2,17 @@ documentcloud/__init__.py,sha256=XAwOR6JYL-flQV_uC616AMA2rYiXTkeogNolqE6LzN4,220
 documentcloud/addon.py,sha256=BvELxbc5pm7vYxo8bWY1VLLcn-VJAQQwmy8Y-G7n26c,12402
 documentcloud/annotations.py,sha256=wVe3wYzyTRvc_hJ3r0m6iyDf6WIFlaGcCnyah_r53pg,2538
 documentcloud/base.py,sha256=pNF45aleYpQ9fj75CiL3c4Ssv6MO1EmdzZ6wBLPKHDg,6545
-documentcloud/client.py,sha256=WXHNE1BT-LE2E55XlOvPuWl_g5N0zUIdXvB7Qj_fMNc,1658
+documentcloud/client.py,sha256=aLFncIVEnN9M3DN0SwL-t8jWIY9CR3Tr7AwSzEHHsP8,3542
 documentcloud/constants.py,sha256=h6NStSkxPrjQ2gzaIlqftCF7tthkRimddOE8SsmlHag,1828
-documentcloud/documents.py,sha256=TlW5fYNEBJQEONVzdVLQTwaoPZaBTBIBRLvrHMevQAA,18230
+documentcloud/documents.py,sha256=sixDwg0cqwv0c45vIL4MKd3uyPt987e9t7GgqjBeh4k,19307
 documentcloud/exceptions.py,sha256=Fq_v7QBcvj-l4yeT7ii_1MrGAPiRs8e1Fwz8qtB4Xqc,344
 documentcloud/organizations.py,sha256=_Ot6MWzoa5JdU3jqedU-0Fec_K8WrgxqdlIp4oIijes,392
 documentcloud/projects.py,sha256=KuOiw65a-8fdgbjo7BqjbEbWguds8inkhFJZJd578bs,5328
 documentcloud/sections.py,sha256=cMf973KMvp6fAPSMXCD67L32Pz1_Tfh81oV2q2UQ9Uk,924
 documentcloud/toolbox.py,sha256=zFZTyOn40YZjBpqa1H3qjpR4C3Wu1X2g72AvH_ljlic,1835
 documentcloud/users.py,sha256=yydOXoEsfJlYqryZpXQ4G3aeRc5y_QCHqXd0dfF1aIc,354
-python_documentcloud-4.6.0.dist-info/licenses/LICENSE,sha256=Z1IBhHCzIeGR9F2iHtcLt2I2qoUhJ2pK139CAIAuFgo,1151
-python_documentcloud-4.6.0.dist-info/METADATA,sha256=A5tJzw613JXb60SaHESdX0L46ryt31XzWIf6VsEx9sA,2880
-python_documentcloud-4.6.0.dist-info/WHEEL,sha256=TdQ5LtNwLuxTCjgxN51AgdU5w-KkB9ttmLbzjTH02pg,109
-python_documentcloud-4.6.0.dist-info/top_level.txt,sha256=rzNW2vA9GqU5ipNQYSP1XJQ54ippjKXVIo9oMlM0Tm4,14
-python_documentcloud-4.6.0.dist-info/RECORD,,
+python_documentcloud-4.7.0.dist-info/licenses/LICENSE,sha256=Z1IBhHCzIeGR9F2iHtcLt2I2qoUhJ2pK139CAIAuFgo,1151
+python_documentcloud-4.7.0.dist-info/METADATA,sha256=QY7JfYCSWmqRz9amTLlZFedeGSw5OZuLqsTaOBcxEgs,2953
+python_documentcloud-4.7.0.dist-info/WHEEL,sha256=TdQ5LtNwLuxTCjgxN51AgdU5w-KkB9ttmLbzjTH02pg,109
+python_documentcloud-4.7.0.dist-info/top_level.txt,sha256=rzNW2vA9GqU5ipNQYSP1XJQ54ippjKXVIo9oMlM0Tm4,14
+python_documentcloud-4.7.0.dist-info/RECORD,,