mostlyright-data 0.16.1__tar.gz → 0.16.2__tar.gz

{mostlyright_data-0.16.1 → mostlyright_data-0.16.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mostlyright-data
-Version: 0.16.1
+Version: 0.16.2
 Summary: Mostly Right CLI for reproducible, reviewed Parquet datasets
 Project-URL: Homepage, https://mostlyright.md/
 Project-URL: Documentation, https://mostlyright.md/docs/guides/cli/

{mostlyright_data-0.16.1 → mostlyright_data-0.16.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mostlyright-data"
-version = "0.16.1"
+version = "0.16.2"
 description = "Mostly Right CLI for reproducible, reviewed Parquet datasets"
 readme = "README.md"
 requires-python = ">=3.13,<3.14"

{mostlyright_data-0.16.1 → mostlyright_data-0.16.2}/src/mostlyright/data_harness/acquisition/http.py

@@ -7,6 +7,7 @@ import errno
 import hashlib
 import http.client
 import json
+import math
 import os
 import re
 import socket
@@ -1479,6 +1480,7 @@ class AcquisitionLimiter:
         self,
         *,
         max_concurrency: int,
+        max_concurrency_per_host: int | None = None,
         min_interval_seconds: float,
         clock: Callable[[], float] = time.monotonic,
         sleep: Callable[[float], None] = time.sleep,
@@ -1493,11 +1495,21 @@ class AcquisitionLimiter:
                 "min_interval_seconds must be in [0, 3600]",
             )
         self._semaphore = threading.BoundedSemaphore(max_concurrency)
+        if max_concurrency_per_host is not None and (
+            type(max_concurrency_per_host) is not int
+            or not 1 <= max_concurrency_per_host <= max_concurrency
+        ):
+            raise AcquisitionSecurityError(
+                "CONCURRENCY_LIMIT",
+                "max_concurrency_per_host must be an integer in [1, max_concurrency]",
+            )
+        self._max_concurrency_per_host = max_concurrency_per_host
         self._min_interval_seconds = float(min_interval_seconds)
         self._clock = clock
         self._sleep = sleep
         self._lock = threading.Lock()
         self._last_request: dict[str, float] = {}
+        self._active_by_host: dict[str, int] = {}
 
     @property
     def clock(self) -> Callable[[], float]:
@@ -1509,11 +1521,10 @@ class AcquisitionLimiter:
     def acquire(self, hostname: str, *, deadline: float) -> Iterator[None]:
         """Hold a concurrency slot, waiting out this host's minimum interval first.
 
-        Two limits, two dispositions, deliberately.  An exhausted concurrency semaphore
-        fails fast: more work is in flight than this process admitted, which is a bug on our
-        side, and waiting would hide it.  The per-host interval waits: it is politeness
-        towards a source that is behaving exactly as documented, and raising there turns a
-        healthy source into a failed run.
+        The global concurrency semaphore fails fast: more work is in flight than this process
+        admitted, which is a bug on our side. An optional per-host concurrency cap and the
+        per-host interval wait: both describe a healthy source boundary, and an immediate
+        refusal would turn temporary occupancy into a failed run.
 
         ``deadline`` is an absolute instant on this limiter's own clock -- the retrieval
         passes its start plus the ``total_timeout_seconds`` it already measures itself
@@ -1523,17 +1534,40 @@ class AcquisitionLimiter:
         up front as ``TOTAL_TIMEOUT`` rather than slept past and regretted afterwards.
         """
 
-        if type(deadline) not in {int, float} or deadline != deadline:  # NaN is not a bound
+        if type(deadline) not in {int, float} or not math.isfinite(float(deadline)):
             raise AcquisitionSecurityError(
                 "TOTAL_TIMEOUT",
                 "a wait must be bounded by a deadline on the retrieval's own clock",
             )
-        if not self._semaphore.acquire(blocking=False):
-            raise AcquisitionSecurityError(
-                "CONCURRENCY_LIMIT",
-                "acquisition concurrency limit is exhausted",
-            )
+        host_slot = False
+        global_slot = False
         try:
+            if self._max_concurrency_per_host is not None:
+                while True:
+                    with self._lock:
+                        active = self._active_by_host.get(hostname, 0)
+                        if active < self._max_concurrency_per_host:
+                            self._active_by_host[hostname] = active + 1
+                            host_slot = True
+                            break
+                    now = float(self._clock())
+                    remaining = float(deadline) - now
+                    if remaining <= 0:
+                        raise AcquisitionSecurityError(
+                            "TOTAL_TIMEOUT",
+                            "the wait for this host's concurrency slot outlived "
+                            "the retrieval budget",
+                        )
+                    # The caller's clock owns the deadline. Keep every individual wait short so
+                    # a sibling's release is observed promptly, while the loop itself remains
+                    # bounded by that same deadline (and test clocks can advance deterministically).
+                    self._sleep(min(remaining, 0.01))
+            if not self._semaphore.acquire(blocking=False):
+                raise AcquisitionSecurityError(
+                    "CONCURRENCY_LIMIT",
+                    "acquisition concurrency limit is exhausted",
+                )
+            global_slot = True
             now = float(self._clock())
             with self._lock:
                 last = self._last_request.get(hostname)
@@ -1553,7 +1587,15 @@ class AcquisitionLimiter:
                 self._sleep(scheduled - now)
             yield
         finally:
-            self._semaphore.release()
+            if global_slot:
+                self._semaphore.release()
+            if host_slot:
+                with self._lock:
+                    remaining = self._active_by_host[hostname] - 1
+                    if remaining:
+                        self._active_by_host[hostname] = remaining
+                    else:
+                        del self._active_by_host[hostname]
 
 
 class PinnedHttpsRetriever:

{mostlyright_data-0.16.1 → mostlyright_data-0.16.2}/src/mostlyright/data_harness/hosted_crawler_fetch.py

@@ -39,7 +39,16 @@ _OPENLIGADB_LIMITER = AcquisitionLimiter(
     max_concurrency=2,
     min_interval_seconds=1.0,
 )
-_PUBLIC_HTTPS_LIMITER = AcquisitionLimiter(max_concurrency=2, min_interval_seconds=1.0)
+# The worker's cgroup scheduler admits at most three public-source children, each of which can
+# issue at most two ranged requests. Eight global slots leave bounded headroom; per-host slots
+# are the provider-facing limit and wait within the retrieval deadline when temporarily occupied.
+PUBLIC_HTTPS_MAX_CONCURRENCY = 8
+PUBLIC_HTTPS_MAX_CONCURRENCY_PER_HOST = 2
+_PUBLIC_HTTPS_LIMITER = AcquisitionLimiter(
+    max_concurrency=PUBLIC_HTTPS_MAX_CONCURRENCY,
+    max_concurrency_per_host=PUBLIC_HTTPS_MAX_CONCURRENCY_PER_HOST,
+    min_interval_seconds=1.0,
+)
 
 # The ranged bulk-fetch envelope for capacity-constrained public origins (EPA AirData is
 # the motivating case: a 60-80 MB archive on a single-IP Apache origin that advertises
@@ -162,7 +171,7 @@ def _public_https_retriever(
             connect_timeout_seconds=5.0,
             read_timeout_seconds=read_timeout_seconds,
             total_timeout_seconds=total_timeout_seconds,
-            max_concurrency=2,
+            max_concurrency=PUBLIC_HTTPS_MAX_CONCURRENCY,
             min_interval_seconds=1.0,
             allowed_media_types=_public_expected_media_types(request.query),
             user_agent=PUBLIC_HTTPS_USER_AGENT,
@@ -236,7 +245,7 @@ def _revalidation_retriever(hostname: str) -> PinnedHttpsRetriever:
             connect_timeout_seconds=5.0,
             read_timeout_seconds=REVALIDATION_READ_TIMEOUT_SECONDS,
             total_timeout_seconds=REVALIDATION_TOTAL_TIMEOUT_SECONDS,
-            max_concurrency=2,
+            max_concurrency=PUBLIC_HTTPS_MAX_CONCURRENCY,
             min_interval_seconds=1.0,
             allowed_media_types=REVALIDATION_MEDIA_TYPES,
             user_agent=PUBLIC_HTTPS_USER_AGENT,

{mostlyright_data-0.16.1 → mostlyright_data-0.16.2}/src/mostlyright/data_harness/hosted_crawler_protocol.py

@@ -90,7 +90,8 @@ PUBLIC_HTTPS_EGRESS_POLICY: dict[str, Any] = {
     # is the meaningful ceiling for a bulk fetch.
     "read_timeout_milliseconds": 120_000,
     "total_timeout_milliseconds": 600_000,
-    "max_concurrency": 2,
+    "max_concurrency": 8,
+    "max_concurrency_per_host": 2,
     "min_interval_milliseconds": 1_000,
     # A bulk object whose 0-0 probe names a total above threshold_bytes is fetched as
     # consecutive range_bytes slices, at most `parallelism` in flight, each slice under