mostlyright-data 0.16.0__tar.gz → 0.16.2__tar.gz

{mostlyright_data-0.16.0 → mostlyright_data-0.16.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mostlyright-data
-Version: 0.16.0
+Version: 0.16.2
 Summary: Mostly Right CLI for reproducible, reviewed Parquet datasets
 Project-URL: Homepage, https://mostlyright.md/
 Project-URL: Documentation, https://mostlyright.md/docs/guides/cli/

{mostlyright_data-0.16.0 → mostlyright_data-0.16.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mostlyright-data"
-version = "0.16.0"
+version = "0.16.2"
 description = "Mostly Right CLI for reproducible, reviewed Parquet datasets"
 readme = "README.md"
 requires-python = ">=3.13,<3.14"

{mostlyright_data-0.16.0 → mostlyright_data-0.16.2}/src/mostlyright/data_harness/acquisition/http.py

@@ -7,6 +7,7 @@ import errno
 import hashlib
 import http.client
 import json
+import math
 import os
 import re
 import socket
@@ -1479,6 +1480,7 @@ class AcquisitionLimiter:
         self,
         *,
         max_concurrency: int,
+        max_concurrency_per_host: int | None = None,
         min_interval_seconds: float,
         clock: Callable[[], float] = time.monotonic,
         sleep: Callable[[float], None] = time.sleep,
@@ -1493,11 +1495,21 @@ class AcquisitionLimiter:
                 "min_interval_seconds must be in [0, 3600]",
             )
         self._semaphore = threading.BoundedSemaphore(max_concurrency)
+        if max_concurrency_per_host is not None and (
+            type(max_concurrency_per_host) is not int
+            or not 1 <= max_concurrency_per_host <= max_concurrency
+        ):
+            raise AcquisitionSecurityError(
+                "CONCURRENCY_LIMIT",
+                "max_concurrency_per_host must be an integer in [1, max_concurrency]",
+            )
+        self._max_concurrency_per_host = max_concurrency_per_host
         self._min_interval_seconds = float(min_interval_seconds)
         self._clock = clock
         self._sleep = sleep
         self._lock = threading.Lock()
         self._last_request: dict[str, float] = {}
+        self._active_by_host: dict[str, int] = {}
 
     @property
     def clock(self) -> Callable[[], float]:
@@ -1509,11 +1521,10 @@ class AcquisitionLimiter:
     def acquire(self, hostname: str, *, deadline: float) -> Iterator[None]:
         """Hold a concurrency slot, waiting out this host's minimum interval first.
 
-        Two limits, two dispositions, deliberately.  An exhausted concurrency semaphore
-        fails fast: more work is in flight than this process admitted, which is a bug on our
-        side, and waiting would hide it.  The per-host interval waits: it is politeness
-        towards a source that is behaving exactly as documented, and raising there turns a
-        healthy source into a failed run.
+        The global concurrency semaphore fails fast: more work is in flight than this process
+        admitted, which is a bug on our side. An optional per-host concurrency cap and the
+        per-host interval wait: both describe a healthy source boundary, and an immediate
+        refusal would turn temporary occupancy into a failed run.
 
         ``deadline`` is an absolute instant on this limiter's own clock -- the retrieval
         passes its start plus the ``total_timeout_seconds`` it already measures itself
@@ -1523,17 +1534,40 @@ class AcquisitionLimiter:
         up front as ``TOTAL_TIMEOUT`` rather than slept past and regretted afterwards.
         """
 
-        if type(deadline) not in {int, float} or deadline != deadline:  # NaN is not a bound
+        if type(deadline) not in {int, float} or not math.isfinite(float(deadline)):
             raise AcquisitionSecurityError(
                 "TOTAL_TIMEOUT",
                 "a wait must be bounded by a deadline on the retrieval's own clock",
             )
-        if not self._semaphore.acquire(blocking=False):
-            raise AcquisitionSecurityError(
-                "CONCURRENCY_LIMIT",
-                "acquisition concurrency limit is exhausted",
-            )
+        host_slot = False
+        global_slot = False
         try:
+            if self._max_concurrency_per_host is not None:
+                while True:
+                    with self._lock:
+                        active = self._active_by_host.get(hostname, 0)
+                        if active < self._max_concurrency_per_host:
+                            self._active_by_host[hostname] = active + 1
+                            host_slot = True
+                            break
+                    now = float(self._clock())
+                    remaining = float(deadline) - now
+                    if remaining <= 0:
+                        raise AcquisitionSecurityError(
+                            "TOTAL_TIMEOUT",
+                            "the wait for this host's concurrency slot outlived "
+                            "the retrieval budget",
+                        )
+                    # The caller's clock owns the deadline. Keep every individual wait short so
+                    # a sibling's release is observed promptly, while the loop itself remains
+                    # bounded by that same deadline (and test clocks can advance deterministically).
+                    self._sleep(min(remaining, 0.01))
+            if not self._semaphore.acquire(blocking=False):
+                raise AcquisitionSecurityError(
+                    "CONCURRENCY_LIMIT",
+                    "acquisition concurrency limit is exhausted",
+                )
+            global_slot = True
             now = float(self._clock())
             with self._lock:
                 last = self._last_request.get(hostname)
@@ -1553,7 +1587,15 @@ class AcquisitionLimiter:
                 self._sleep(scheduled - now)
             yield
         finally:
-            self._semaphore.release()
+            if global_slot:
+                self._semaphore.release()
+            if host_slot:
+                with self._lock:
+                    remaining = self._active_by_host[hostname] - 1
+                    if remaining:
+                        self._active_by_host[hostname] = remaining
+                    else:
+                        del self._active_by_host[hostname]
 
 
 class PinnedHttpsRetriever:

{mostlyright_data-0.16.0 → mostlyright_data-0.16.2}/src/mostlyright/data_harness/acquisition/parsing.py

@@ -442,6 +442,215 @@ def parse_csv_evidence(
     )
 
 
+def parse_ndjson_evidence(
+    content: bytes,
+    *,
+    media_type: str,
+    filename: str,
+    limits: ParseLimits,
+) -> ParsedTabularEvidence:
+    """Stream one NDJSON verification pass: the admissions of the drained parse, record by record.
+
+    The second streaming pass, and the reason :data:`STREAMED_EVIDENCE_FORMATS` is a table rather
+    than the word ``csv``.  It serves NDJSON acquired over public or authenticated HTTPS, which is
+    the lane that reaches ``InProcessPublicHttpsAdapter._confined_read``, and a replayed WebSocket
+    capture, which reaches it through ``v4_stream_recording.RecordedStreamAdapter``.  That second
+    caller still needs rows, but only one per sealed batch, so it drains each batch on its own
+    and reads the concatenation through this pass alone.
+
+    WHAT IT ACTUALLY BUYS, MEASURED INSIDE THIS FUNCTION WITH ``tracemalloc``, WHICH IS NOT THE
+    SAME QUANTITY THE BENCHMARK REPORTS.  Every figure in this paragraph is the peak of the Python
+    objects THIS PASS allocates.  It is deliberately not the number
+    ``evidence/acquisition-ceiling-benchmark.macos-arm64.json`` carries: that file records the
+    CHILD'S peak resident set (``ru_maxrss``), which also holds the interpreter, the PyArrow
+    import and the allocator's own slack.  The two registers are reconciled at the end of this
+    paragraph, because reading one as the other is how a later change came to claim a memory win
+    twice the size of the real one.
+
+    No list of decoded records and no tuple of rows is ever built, where the drained
+    ``_parse_json`` holds BOTH at once.  On a four-column order-book shape that is 2.87x the
+    source against 10.95x at 320k records, and the ratio is what moves rather than the growth:
+    this pass is NOT constant in the number of records and saying so would be false.
+    ``text.splitlines()`` materialises every line of the document, so the peak grows linearly with
+    the row count exactly as the drained parse's does -- 2.89x at 50k records and 2.87x at 320k,
+    against 11.12x and 10.95x drained.  It is flat from 50k up rather than at every size: at 5k
+    records the pass's own constant costs are still visible and it reads 3.46x.  The shape moves
+    it too -- the benchmark's seven-column frame reads 2.38x against 9.04x at 320k -- so the pair
+    is quoted together or neither number means anything.
+
+    RECONCILING THE TWO REGISTERS, ONCE, HERE.  Function-level 2.87x and the child's 7.33x for the
+    same read are both honest: the child's reading adds a large fixed cost that no source byte
+    pays for.  That cost is common to BOTH operations, so it raises both multipliers and
+    COMPRESSES the ratio between them -- this pass is worth 3.8x at the function level and 2.0x
+    measured as the child's peak on the order-book shape.  A caller sizing a confined read wants
+    the child's number; a reader asking what this function holds wants this one.
+
+    THE LINE LIST IS REQUIRED, NOT AN OVERSIGHT.  ``_parse_json`` numbers its records by
+    ``str.splitlines()``, which breaks on U+2028, U+0085 and the vertical tab as well as on the
+    newline.  A reader that split the bytes on ``\n`` would find a different number of records in
+    the same document and refuse at different record numbers, which is the parity this pass exists
+    to hold.  So the trade is deliberate: the same line list, and just over a quarter of what this
+    function allocates, at a fetch ceiling 2.25 times higher.  Both halves of that trade are in
+    the ``tracemalloc`` register named above; the child-level statement of the same trade is the
+    one in ``sandbox.py`` beside ``STREAMING_OPERATIONS``, and it is the one to quote when the
+    question is what a confined read costs.
+
+    THE REFUSALS ARE ``_parse_json``'s OWN, AT THE SAME RECORDS, IN THE SAME ORDER, and preserving
+    that order is most of this function.  The drained parse reads every line into a list before it
+    asks a single question about shape, so its refusals fall in four groups that do not interleave:
+    the per-line group (the row budget, then a line that is not JSON), then the whole-document
+    group (an empty document, then a member that is not an object), then the per-record group (the
+    nesting budget, the first-record schema, the field budget), then the table group (the cell
+    budget).  A pass that raised each fault where it found it would report the SECOND group's
+    fault only when no later line carried a FIRST group's, and would report a third-group fault
+    that the drained parse never reaches at all.
+
+    So only the per-line group raises where it is found -- it is the group the drained parse also
+    resolves before anything else -- and every later group is recorded and raised after the last
+    line has been read, in the drained parse's own order.  That is the same deferral
+    :func:`parse_csv_evidence` applies to the total-cell budget, applied to three more groups
+    because NDJSON has three more questions that are asked about the document rather than the row.
+
+    The row digest is fed to the hash incrementally, by the framing
+    :func:`row_digest_for` uses, for the reason stated there.
+    """
+
+    _admit_tabular_input(content, data_format="ndjson", media_type=media_type, filename=filename)
+    if not content or len(content) > limits.max_input_bytes:
+        raise AcquisitionSecurityError(
+            "PARSE_INPUT_LIMIT",
+            "source bytes are empty or exceed the parser input budget",
+        )
+    if any(content.startswith(magic) for magic in _ARCHIVE_OR_EXECUTABLE_MAGICS):
+        raise AcquisitionSecurityError(
+            "PARSE_FORBIDDEN_CONTAINER",
+            "archives, executables, documents, and nested containers are forbidden",
+        )
+    # No NUL check, deliberately.  ``_parse_csv`` makes one and ``_parse_json`` does not, so a
+    # NUL inside an NDJSON string reaches the decoder in the drained parse and must reach it
+    # here; adding the CSV check would refuse a document the drained read admits.
+    text = _strict_utf8(content)
+    decoder = json.JSONDecoder(
+        object_pairs_hook=_unique_object,
+        parse_constant=lambda value: _invalid_json_number(value),
+    )
+
+    columns: tuple[str, ...] = ()
+    expected: set[str] = set()
+    observed_types: list[set[str]] = []
+    row_hash = hashlib.sha256()
+    row_count = 0
+    hashed_rows = 0
+    # The deferred groups, in the order the drained parse resolves them.  ``not_an_object`` and
+    # an exhausted loop with ``row_count`` still zero are the whole-document group; the two
+    # ``AcquisitionSecurityError`` slots hold a refusal that is RE-RAISED rather than rebuilt, so
+    # the sentence a person is told is the drained parse's own down to the record number in it.
+    not_an_object = False
+    no_columns: AcquisitionSecurityError | None = None
+    record_refusal: AcquisitionSecurityError | None = None
+    over_cell_budget = False
+    canonical_error: CanonicalJSONError | None = None
+
+    for line_number, line in enumerate(text.splitlines(), start=1):
+        if not line.strip():
+            continue
+        if row_count >= limits.max_rows:
+            raise AcquisitionSecurityError("PARSE_ROW_LIMIT", "NDJSON exceeds the row budget")
+        try:
+            record = decoder.decode(line)
+        except json.JSONDecodeError:
+            raise AcquisitionSecurityError(
+                "PARSE_JSON",
+                f"NDJSON line {line_number} is malformed",
+            ) from None
+        row_count += 1
+        if not isinstance(record, dict):
+            not_an_object = True
+        # Nothing below this line can win against a fault already recorded above it, so the work
+        # is skipped rather than done and discarded.  That is what keeps the deferral from costing
+        # a full validation pass over a document whose first question already has an answer.
+        if not_an_object or no_columns is not None or record_refusal is not None:
+            continue
+        if not columns:
+            try:
+                columns = _columns(list(record), limits)
+            except AcquisitionSecurityError as error:
+                no_columns = error
+                continue
+            expected = set(columns)
+            observed_types = [set() for _ in columns]
+            row_hash.update(b'{"columns":' + canonical_json_bytes(list(columns)) + b',"rows":[')
+        try:
+            _validate_json_value(record, depth=1, limits=limits)
+            if set(record) != expected:
+                raise AcquisitionSecurityError(
+                    "PARSE_ROW_SCHEMA",
+                    f"JSON row {row_count} does not match the first-row schema",
+                )
+            row = tuple(record[column] for column in columns)
+            _validate_row(row, limits)
+        except AcquisitionSecurityError as error:
+            record_refusal = error
+            continue
+        if row_count * len(columns) > limits.max_total_cells:
+            over_cell_budget = True
+        # A value the canonical encoder refuses keeps the position the one-shot digest gave it:
+        # after every parse refusal and the cell budget, never mid-loop.  The hash is dead on that
+        # path, so it stops accumulating with the first error.
+        if canonical_error is None:
+            try:
+                encoded_row = canonical_json_bytes(list(row))
+            except CanonicalJSONError as error:
+                canonical_error = error
+            else:
+                if hashed_rows:
+                    row_hash.update(b",")
+                row_hash.update(encoded_row)
+                hashed_rows += 1
+        for index, value in enumerate(row):
+            observed_types[index].add("null" if value is None else type(value).__name__)
+
+    if not_an_object:
+        raise AcquisitionSecurityError("PARSE_JSON_SHAPE", "every JSON row must be an object")
+    if not row_count:
+        raise AcquisitionSecurityError(
+            "PARSE_JSON_SHAPE",
+            "JSON tabular input must be a non-empty array of objects",
+        )
+    if no_columns is not None:
+        raise no_columns
+    if record_refusal is not None:
+        raise record_refusal
+    if over_cell_budget:
+        raise AcquisitionSecurityError("PARSE_CELL_LIMIT", "table exceeds the cell budget")
+    if canonical_error is not None:
+        # TYPED, LIKE EVERY OTHER PARSE REFUSAL HERE, and for the reason ``parse_csv_evidence``
+        # states at its own copy of this branch: raised bare it leaves this boundary as an
+        # exception nothing above can name.
+        raise AcquisitionSecurityError(
+            "PARSE_VALUE", "parsed field contains a non-canonical value"
+        ) from canonical_error
+    row_hash.update(b"]}")
+
+    return ParsedTabularEvidence(
+        data_format="ndjson",
+        input_sha256=sha256_bytes(content),
+        columns=columns,
+        row_count=row_count,
+        schema_digest=canonical_sha256(
+            {
+                "data_format": "ndjson",
+                "columns": list(columns),
+                "types": [
+                    "|".join(sorted(observed)) if observed else "empty"
+                    for observed in observed_types
+                ],
+            }
+        ),
+        row_digest=row_hash.hexdigest(),
+    )
+
+
 def parse_csv_evidence_to_file(
     source: BinaryIO,
     destination: BinaryIO,

{mostlyright_data-0.16.0 → mostlyright_data-0.16.2}/src/mostlyright/data_harness/acquisition/sandbox.py

@@ -53,6 +53,7 @@ from mostlyright.data_harness.acquisition.parsing import (
     ParseLimits,
     parse_csv_evidence,
     parse_csv_evidence_to_file,
+    parse_ndjson_evidence,
     parse_streaming_csv_window,
     parse_tabular_bytes,
     prime_parquet_support,
@@ -147,6 +148,9 @@ SANDBOX_OPERATIONS = frozenset(
 # bounded by the number of columns rather than by the size of the source.  ``parse`` holds both
 # and is measured at roughly twenty-one times the source bytes for it; the streaming pass is
 # measured at roughly eight, and that difference is the entire reason this operation exists.
+# ``parse_evidence`` reads two encodings, and the set is still per operation rather than per
+# encoding: the budgets below bound what a CHILD may be asked for, and a child asked for the
+# streaming read holds one record whichever pass reads it.
 #
 # It is a set rather than a flag because two independent budgets read it -- the per-operation
 # request ceiling below and the container-boundary admission in ``_invoke`` -- and a second
@@ -2358,14 +2362,36 @@ def _execute_worker_request(request: dict[str, Any]) -> dict[str, Any]:
             # The streaming pass, and the whole point of the operation: the same admissions and
             # the same refusals at the same rows as the drained parse, holding one row at a time.
             # The table is never built, so nothing here is released later -- there is nothing to
-            # release. ``data_format`` was already held to this operation's attested formats
-            # above, and only CSV is attested, so the CSV pass is the whole of what may run.
-            evidence = parse_csv_evidence(
-                content,
-                media_type=media_type,
-                filename=filename,
-                limits=limits,
-            )
+            # release.
+            #
+            # TWO PASSES NOW, AND THE DISPATCH IS EXHAUSTIVE OVER THE ATTESTED LIST RATHER THAN A
+            # DEFAULT.  ``data_format`` was already held to this operation's attested formats
+            # above, so the branch below can only be reached by a name
+            # ``STREAMED_EVIDENCE_FORMATS`` admits; an ``else`` that fell through to one of the
+            # two passes would hand a third encoding's bytes to whichever pass was written last,
+            # and the evidence it returned would attest a reader that never ran. That is the same
+            # refusal ``_parse_tabular`` makes for the same reason, and it is a loud failure until
+            # a pass is written rather than a silent misparse.
+            if data_format == "csv":
+                evidence = parse_csv_evidence(
+                    content,
+                    media_type=media_type,
+                    filename=filename,
+                    limits=limits,
+                )
+            elif data_format == "ndjson":
+                evidence = parse_ndjson_evidence(
+                    content,
+                    media_type=media_type,
+                    filename=filename,
+                    limits=limits,
+                )
+            else:
+                raise AcquisitionSecurityError(
+                    "PARSE_FORMAT_UNHANDLED",
+                    f"data format {data_format!r} is attested for the streaming read but no "
+                    "streaming pass claims it",
+                )
             base["evidence"] = _evidence_dict(evidence)
             return base
         parsed = parse_tabular_bytes(

{mostlyright_data-0.16.0 → mostlyright_data-0.16.2}/src/mostlyright/data_harness/formats.py

@@ -71,17 +71,43 @@ READER_WORKER_MEMORY_BYTES = 512 * 1024 * 1024
 # families, not encodings a parser is ever handed.
 READER_OUTPUT_FORMATS: tuple[str, ...] = ("csv",)
 
-# Encodings a confined interpretation may read one row at a time.
+# Encodings the INLINE confined interpretation may read one row at a time.
 #
 # This is a third question, and it must not be answered by either table above.  ``DATA_FORMATS``
 # says what may arrive on the wire and ``READER_OUTPUT_FORMATS`` says what a decoder may emit;
-# this one says what the streaming verification pass in ``acquisition.parsing`` can actually
-# read without materialising the table.  Only the CSV pass exists, so this is one value -- and
-# it is spelled separately from ``READER_OUTPUT_FORMATS`` because the two are equal today by
-# coincidence rather than by rule.  Writing a streaming JSON pass would move this table alone;
-# adding a Reader output encoding would move that one alone.  Sharing the literal would rotate
-# the wrong operation's attestation on either event.
-STREAMED_EVIDENCE_FORMATS: tuple[str, ...] = ("csv",)
+# this one says what the streaming verification passes in ``acquisition.parsing`` can actually
+# read without materialising the table.  It is spelled separately from ``READER_OUTPUT_FORMATS``
+# because the two were equal by coincidence rather than by rule, and NDJSON is the event that
+# separated them.
+#
+# WHICH LANES THIS SERVES.  NDJSON acquired over public or authenticated HTTPS -- the sources that
+# reach ``InProcessPublicHttpsAdapter._confined_read`` -- and, since the recording adapter was
+# taught to order both reads, a replayed WebSocket capture as well.  A Recorder seals one frame to
+# a line, so ``v4_stream_recording.RecordedStreamAdapter`` reads the whole concatenation through
+# this table's NDJSON pass for the facts a receipt states, and drains ONE SEALED BATCH AT A TIME
+# for the frame each batch's ``recorder_position`` is spelled out of.  Its confined peak is
+# therefore a function of the largest batch rather than of the recording, which is what moved the
+# order-book memory ceiling.  Both lanes order the same pair, so both pin the same composition.
+#
+# THIS TABLE IS DIGEST-BEARING AND ITS SCOPE IS THE INLINE READ ALONE.  It is embedded in the
+# policy document of ``parse_evidence`` through ``PARSER_FORMATS_BY_OPERATION`` below, so adding
+# to it rotates that operation's attestation -- which is the intended cost of teaching the inline
+# pass a second encoding.  It used to be the value of three registry entries, and widening it
+# then would have rotated three attestations to describe one new pass: the two descriptor-backed
+# operations read CSV through ``csv.reader`` and gain nothing from an encoding they cannot parse.
+# They now name ``STREAMED_DESCRIPTOR_FORMATS`` instead, so each table moves the operations it
+# actually describes.
+STREAMED_EVIDENCE_FORMATS: tuple[str, ...] = ("csv", "ndjson")
+
+# Encodings the DESCRIPTOR-BACKED confined interpretations may read one row at a time.
+#
+# Split from the table above rather than shared with it, because the two answer different
+# questions about different code.  ``parse_evidence_file`` and ``parse_streaming_csv_window``
+# take an opened descriptor and drive ``csv.reader`` over a strict-UTF-8 text wrapper; there is
+# no NDJSON branch behind either, and a format list that admitted one would attest a pass that
+# does not exist.  Writing a descriptor-backed NDJSON pass would move this table alone, exactly
+# as writing the inline one moved the table above alone.
+STREAMED_DESCRIPTOR_FORMATS: tuple[str, ...] = ("csv",)
 
 # The ordered parser table.  Order is digest-load-bearing: this exact sequence is
 # embedded in the sandbox policy document built by
@@ -294,8 +320,8 @@ PARSER_FORMATS_BY_OPERATION: Mapping[str, tuple[str, ...]] = MappingProxyType(
     {
         "parse": PARSER_FORMAT_ORDER,
         "parse_evidence": STREAMED_EVIDENCE_FORMATS,
-        "parse_evidence_file": STREAMED_EVIDENCE_FORMATS,
-        "parse_streaming_csv_window": STREAMED_EVIDENCE_FORMATS,
+        "parse_evidence_file": STREAMED_DESCRIPTOR_FORMATS,
+        "parse_streaming_csv_window": STREAMED_DESCRIPTOR_FORMATS,
         "retrieve_and_parse": PARSER_FORMAT_ORDER,
         "decode_and_parse": READER_OUTPUT_FORMATS,
         "retrieve_decode_and_parse": READER_OUTPUT_FORMATS,

{mostlyright_data-0.16.0 → mostlyright_data-0.16.2}/src/mostlyright/data_harness/hosted_crawler_fetch.py

@@ -39,7 +39,16 @@ _OPENLIGADB_LIMITER = AcquisitionLimiter(
     max_concurrency=2,
     min_interval_seconds=1.0,
 )
-_PUBLIC_HTTPS_LIMITER = AcquisitionLimiter(max_concurrency=2, min_interval_seconds=1.0)
+# The worker's cgroup scheduler admits at most three public-source children, each of which can
+# issue at most two ranged requests. Eight global slots leave bounded headroom; per-host slots
+# are the provider-facing limit and wait within the retrieval deadline when temporarily occupied.
+PUBLIC_HTTPS_MAX_CONCURRENCY = 8
+PUBLIC_HTTPS_MAX_CONCURRENCY_PER_HOST = 2
+_PUBLIC_HTTPS_LIMITER = AcquisitionLimiter(
+    max_concurrency=PUBLIC_HTTPS_MAX_CONCURRENCY,
+    max_concurrency_per_host=PUBLIC_HTTPS_MAX_CONCURRENCY_PER_HOST,
+    min_interval_seconds=1.0,
+)
 
 # The ranged bulk-fetch envelope for capacity-constrained public origins (EPA AirData is
 # the motivating case: a 60-80 MB archive on a single-IP Apache origin that advertises
@@ -162,7 +171,7 @@ def _public_https_retriever(
             connect_timeout_seconds=5.0,
             read_timeout_seconds=read_timeout_seconds,
             total_timeout_seconds=total_timeout_seconds,
-            max_concurrency=2,
+            max_concurrency=PUBLIC_HTTPS_MAX_CONCURRENCY,
             min_interval_seconds=1.0,
             allowed_media_types=_public_expected_media_types(request.query),
             user_agent=PUBLIC_HTTPS_USER_AGENT,
@@ -236,7 +245,7 @@ def _revalidation_retriever(hostname: str) -> PinnedHttpsRetriever:
             connect_timeout_seconds=5.0,
             read_timeout_seconds=REVALIDATION_READ_TIMEOUT_SECONDS,
             total_timeout_seconds=REVALIDATION_TOTAL_TIMEOUT_SECONDS,
-            max_concurrency=2,
+            max_concurrency=PUBLIC_HTTPS_MAX_CONCURRENCY,
             min_interval_seconds=1.0,
             allowed_media_types=REVALIDATION_MEDIA_TYPES,
             user_agent=PUBLIC_HTTPS_USER_AGENT,

{mostlyright_data-0.16.0 → mostlyright_data-0.16.2}/src/mostlyright/data_harness/hosted_crawler_protocol.py

@@ -90,7 +90,8 @@ PUBLIC_HTTPS_EGRESS_POLICY: dict[str, Any] = {
     # is the meaningful ceiling for a bulk fetch.
     "read_timeout_milliseconds": 120_000,
     "total_timeout_milliseconds": 600_000,
-    "max_concurrency": 2,
+    "max_concurrency": 8,
+    "max_concurrency_per_host": 2,
     "min_interval_milliseconds": 1_000,
     # A bulk object whose 0-0 probe names a total above threshold_bytes is fetched as
     # consecutive range_bytes slices, at most `parallelism` in flight, each slice under