python-documentcloud 4.5.0__py2.py3-none-any.whl → 4.7.0__py2.py3-none-any.whl

documentcloud/addon.py

@@ -182,6 +182,26 @@ class AddOn(BaseAddOn):
             f"addon_runs/{self.id}/", json={"file_name": file_name}
         )
 
+    def load_run_data(self):
+        "Load persistent data from this run"
+        if not self.id:
+            return {}
+
+        response = self.client.get(f"addon_runs/{self.id}/")
+        response.raise_for_status()
+        return response.json().get("data", {})
+
+    def store_run_data(self, data):
+        "Store persistent data for this run"
+        if not self.id:
+            print("Run ID not set. Try again later or check if something went wrong.")
+            return None
+
+        if not isinstance(data, dict):
+            raise TypeError("Invalid data")
+
+        return self.client.patch(f"addon_runs/{self.id}/", json={"data": data})
+
     def load_event_data(self):
         """Load persistent data for this event"""
         if not self.event_id:

documentcloud/client.py

@@ -1,12 +1,12 @@
-# Import SquareletClient from python-squarelet
 # Standard Library
 import logging
+import time
 
 # Third Party
+import token_bucket
 from squarelet import SquareletClient
 
 # Local
-# Local Imports
 from .documents import DocumentClient
 from .organizations import OrganizationClient
 from .projects import ProjectClient
@@ -14,6 +14,22 @@ from .users import UserClient
 
 logger = logging.getLogger("documentcloud")
 
+# Per-endpoint rate limits applied on top of the global squarelet limit.
+# Format: (method, url_pattern, rate_per_second, capacity)
+#
+# Endpoint              Rate        Burst   Notes
+# --------              ----        -----   -----
+# GET  documents/search  15/min      50
+# POST documents/        12/min      100     25 docs/bulk call = up to 300 docs/min
+# PUT  documents/        12/min      100     25 docs/bulk call = up to 300 docs/min
+# GET  files/            15/min      100     PDFs, full text, and other private assets
+ENDPOINT_RATE_LIMITS = [
+    ("GET", "documents/search", 15 / 60, 50),
+    ("POST", "documents/", 12 / 60, 100),
+    ("PUT", "documents/", 12 / 60, 100),
+    ("GET", "files/", 15 / 60, 100),
+]
+
 
 class DocumentCloud(SquareletClient):
     """
@@ -51,8 +67,34 @@ class DocumentCloud(SquareletClient):
         else:
             logger.addHandler(logging.NullHandler())
 
+        # Build per-endpoint token bucket rate limiters
+        storage = token_bucket.MemoryStorage()
+        self._endpoint_limiters = [
+            (
+                pattern_method,
+                pattern,
+                token_bucket.Limiter(rate=rate, capacity=capacity, storage=storage),
+                f"{pattern_method}:{pattern}",
+            )
+            for pattern_method, pattern, rate, capacity in ENDPOINT_RATE_LIMITS
+        ]
+
         # Initialize the sub-clients using SquareletClient
         self.documents = DocumentClient(self)
         self.projects = ProjectClient(self)
         self.users = UserClient(self)
         self.organizations = OrganizationClient(self)
+
+    def request(self, method, url, raise_error=True, **kwargs):
+        for pattern_method, pattern, limiter, bucket_key in self._endpoint_limiters:
+            if pattern_method.upper() == method.upper() and pattern in url:
+                if not limiter.consume(bucket_key):
+                    logger.warning(
+                        "Rate limit reached for %s %s, throttling...",
+                        method.upper(),
+                        pattern,
+                    )
+                    while not limiter.consume(bucket_key):
+                        time.sleep(0.1)
+                return super().request(method, url, raise_error=raise_error, **kwargs)
+        return super().request(method, url, raise_error=raise_error, **kwargs)

documentcloud/documents.py

@@ -7,10 +7,13 @@ import datetime
 import logging
 import os
 import re
+import time
 import warnings
 from functools import partial
+from urllib.parse import urlparse
 
 # Third Party
+import token_bucket
 from requests.exceptions import RequestException
 
 # Local
@@ -23,15 +26,12 @@ from .sections import SectionClient
 from .toolbox import grouper, is_url, merge_dicts, requests_retry_session
 from .users import User
 
-try:
-    from urllib.parse import urlparse
-except ImportError:
-    from urlparse import urlparse
-
 logger = logging.getLogger("documentcloud")
 
 IMAGE_SIZES = ["thumbnail", "small", "normal", "large", "xlarge"]
 
+DEFAULT_USER_AGENT = "python-documentcloud"
+
 
 class Document(BaseAPIObject):
     """A single DocumentCloud document"""
@@ -168,12 +168,17 @@ class Document(BaseAPIObject):
 
         if base_netloc == url_netloc:
             # if the url host is the same as the base api host,
-            # sent the request with the client in order to include
+            # send the request with the client in order to include
             # authentication credentials
             response = self._client.get(url, full_url=True)
         else:
-            response = requests_retry_session().get(
-                url, headers={"User-Agent": "python-documentcloud2"}
+            response = self._client.documents.asset_get(
+                url,
+                headers={
+                    "User-Agent": self._client.session.headers.get(
+                        "User-Agent", DEFAULT_USER_AGENT
+                    )
+                },
             )
         if fmt == "text":
             return response.content.decode("utf8")
@@ -250,6 +255,26 @@ class DocumentClient(BaseAPIClient):
     api_path = "documents"
     resource = Document
 
+    def __init__(self, client):
+        super().__init__(client)
+        # Rate limit for public document asset fetches (S3-hosted).
+        # Private document assets go through the API client and are limited there.
+        # Token bucket: burst of 100, sustained at 15/min (0.25/sec).
+        storage = token_bucket.MemoryStorage()
+        self._asset_limiter = token_bucket.Limiter(
+            rate=15 / 60,
+            capacity=100,
+            storage=storage,
+        )
+        self._asset_session = requests_retry_session()
+
+    def asset_get(self, url, **kwargs):
+        if not self._asset_limiter.consume("asset"):
+            logger.warning("Rate limit reached for asset fetch, throttling...")
+            while not self._asset_limiter.consume("asset"):
+                time.sleep(0.1)
+        return self._asset_session.get(url, **kwargs)
+
     def search(self, query, **params):
         """Return documents matching a search query"""
 

documentcloud/exceptions.py

@@ -2,11 +2,14 @@
 Custom exceptions for python-documentcloud
 """
 
+# Third Party
 # pylint: disable=unused-import
 # Import exceptions from python-squarelet
-from squarelet.exceptions import SquareletError as DocumentCloudError
-from squarelet.exceptions import DuplicateObjectError
-from squarelet.exceptions import CredentialsFailedError
-from squarelet.exceptions import APIError
-from squarelet.exceptions import DoesNotExistError
-from squarelet.exceptions import MultipleObjectsReturnedError
+from squarelet.exceptions import (
+    APIError,
+    CredentialsFailedError,
+    DoesNotExistError,
+    DuplicateObjectError,
+    MultipleObjectsReturnedError,
+    SquareletError as DocumentCloudError,
+)

{python_documentcloud-4.5.0.dist-info → python_documentcloud-4.7.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-documentcloud
-Version: 4.5.0
+Version: 4.7.0
 Summary: A simple Python wrapper for the DocumentCloud API
 Home-page: https://github.com/muckrock/python-documentcloud
 Author: Mitchell Kotler
@@ -27,6 +27,7 @@ Requires-Dist: urllib3
 Requires-Dist: pyyaml
 Requires-Dist: fastjsonschema
 Requires-Dist: python-squarelet
+Requires-Dist: token-bucket
 Provides-Extra: dev
 Requires-Dist: black; extra == "dev"
 Requires-Dist: coverage; extra == "dev"
@@ -37,6 +38,7 @@ Requires-Dist: twine; extra == "dev"
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: pytest-mock; extra == "test"
+Requires-Dist: pytest-xdist; extra == "test"
 Requires-Dist: pytest-recording; extra == "test"
 Requires-Dist: vcrpy; extra == "test"
 Dynamic: author

{python_documentcloud-4.5.0.dist-info → python_documentcloud-4.7.0.dist-info}/RECORD

@@ -1,18 +1,18 @@
 documentcloud/__init__.py,sha256=XAwOR6JYL-flQV_uC616AMA2rYiXTkeogNolqE6LzN4,220
-documentcloud/addon.py,sha256=3FxQjm26jknjLdd-GuztiZO4Z7NcgXq4WqunE9oh2es,11754
+documentcloud/addon.py,sha256=BvELxbc5pm7vYxo8bWY1VLLcn-VJAQQwmy8Y-G7n26c,12402
 documentcloud/annotations.py,sha256=wVe3wYzyTRvc_hJ3r0m6iyDf6WIFlaGcCnyah_r53pg,2538
 documentcloud/base.py,sha256=pNF45aleYpQ9fj75CiL3c4Ssv6MO1EmdzZ6wBLPKHDg,6545
-documentcloud/client.py,sha256=WXHNE1BT-LE2E55XlOvPuWl_g5N0zUIdXvB7Qj_fMNc,1658
+documentcloud/client.py,sha256=aLFncIVEnN9M3DN0SwL-t8jWIY9CR3Tr7AwSzEHHsP8,3542
 documentcloud/constants.py,sha256=h6NStSkxPrjQ2gzaIlqftCF7tthkRimddOE8SsmlHag,1828
-documentcloud/documents.py,sha256=dgoUr2XsxYmxC1xv3lJHgFQdJyE_rBNa2QS0Mn5Y2Is,18294
-documentcloud/exceptions.py,sha256=AwIJpcylq6sF6oaenrZE6nr2EBuj23nxTOf3z_RwtuE,461
+documentcloud/documents.py,sha256=sixDwg0cqwv0c45vIL4MKd3uyPt987e9t7GgqjBeh4k,19307
+documentcloud/exceptions.py,sha256=Fq_v7QBcvj-l4yeT7ii_1MrGAPiRs8e1Fwz8qtB4Xqc,344
 documentcloud/organizations.py,sha256=_Ot6MWzoa5JdU3jqedU-0Fec_K8WrgxqdlIp4oIijes,392
 documentcloud/projects.py,sha256=KuOiw65a-8fdgbjo7BqjbEbWguds8inkhFJZJd578bs,5328
 documentcloud/sections.py,sha256=cMf973KMvp6fAPSMXCD67L32Pz1_Tfh81oV2q2UQ9Uk,924
 documentcloud/toolbox.py,sha256=zFZTyOn40YZjBpqa1H3qjpR4C3Wu1X2g72AvH_ljlic,1835
 documentcloud/users.py,sha256=yydOXoEsfJlYqryZpXQ4G3aeRc5y_QCHqXd0dfF1aIc,354
-python_documentcloud-4.5.0.dist-info/licenses/LICENSE,sha256=Z1IBhHCzIeGR9F2iHtcLt2I2qoUhJ2pK139CAIAuFgo,1151
-python_documentcloud-4.5.0.dist-info/METADATA,sha256=90GM8QOJIaQfjjZ_KKyxqvkQr1rxKg6IpCnAI_FZS1I,2880
-python_documentcloud-4.5.0.dist-info/WHEEL,sha256=JNWh1Fm1UdwIQV075glCn4MVuCRs0sotJIq-J6rbxCU,109
-python_documentcloud-4.5.0.dist-info/top_level.txt,sha256=rzNW2vA9GqU5ipNQYSP1XJQ54ippjKXVIo9oMlM0Tm4,14
-python_documentcloud-4.5.0.dist-info/RECORD,,
+python_documentcloud-4.7.0.dist-info/licenses/LICENSE,sha256=Z1IBhHCzIeGR9F2iHtcLt2I2qoUhJ2pK139CAIAuFgo,1151
+python_documentcloud-4.7.0.dist-info/METADATA,sha256=QY7JfYCSWmqRz9amTLlZFedeGSw5OZuLqsTaOBcxEgs,2953
+python_documentcloud-4.7.0.dist-info/WHEEL,sha256=TdQ5LtNwLuxTCjgxN51AgdU5w-KkB9ttmLbzjTH02pg,109
+python_documentcloud-4.7.0.dist-info/top_level.txt,sha256=rzNW2vA9GqU5ipNQYSP1XJQ54ippjKXVIo9oMlM0Tm4,14
+python_documentcloud-4.7.0.dist-info/RECORD,,

{python_documentcloud-4.5.0.dist-info → python_documentcloud-4.7.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: setuptools (82.0.1)
 Root-Is-Purelib: true
 Tag: py2-none-any
 Tag: py3-none-any