classifyre-cli 0.4.82__tar.gz → 0.4.83__tar.gz

classifyre_cli-0.4.83/.turbo/turbo-build.log

@@ -0,0 +1,3 @@
+$ uv sync
+Resolved 307 packages in 237ms
+Checked 51 packages in 6ms

{classifyre_cli-0.4.82 → classifyre_cli-0.4.83}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: classifyre-cli
-Version: 0.4.82
+Version: 0.4.83
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.4.82 → classifyre_cli-0.4.83}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.4.82",
+  "version": "0.4.83",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.4.82 → classifyre_cli-0.4.83}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.4.82"
+version = "0.4.83"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 requires-python = ">=3.12"

{classifyre_cli-0.4.82 → classifyre_cli-0.4.83}/src/models/generated_input.py

@@ -977,13 +977,39 @@ class S3CompatibleStorageOptionalConnection(BaseModel):
     )
     max_object_bytes: int | None = Field(
         5242880,
-        description='Maximum bytes downloaded per object for MIME detection and text extraction',
+        description='Maximum bytes of one object held in memory. Larger objects are streamed to a temporary file (or read by byte range where the provider supports it), so this bounds memory rather than the size of file that can be scanned. See max_file_bytes to refuse large objects outright.',
         ge=1024,
-        le=52428800,
     )
     verify_ssl: bool | None = Field(
         True, description='TLS certificate verification toggle'
     )
+    max_file_bytes: int | None = Field(
+        None,
+        description='Refuse any object larger than this many bytes. 0 or unset means no limit: an object above max_object_bytes is spooled to disk rather than held in memory, so file size is bounded by free disk, not by RAM.',
+        ge=0,
+    )
+    max_archive_members: int | None = Field(
+        200,
+        description='Maximum member files expanded from one archive into child assets. Bounds fan-out, and is the guard against a zip bomb.',
+        ge=1,
+        le=10000,
+    )
+    max_archive_member_bytes: int | None = Field(
+        10485760,
+        description='Maximum uncompressed bytes read from a single archive member',
+        ge=1024,
+    )
+    max_archive_total_bytes: int | None = Field(
+        104857600,
+        description='Maximum uncompressed bytes read across all members of one archive. The decompression-ratio ceiling: a zip bomb hits this before it hits memory.',
+        ge=1024,
+    )
+    max_embedded_files: int | None = Field(
+        200,
+        description='Maximum embedded files (parquet image/audio columns, office media) expanded from one container into child assets per run',
+        ge=1,
+        le=10000,
+    )
 
 
 class S3CompatibleStorageOptional(BaseModel):
@@ -1129,9 +1155,35 @@ class AzureBlobStorageOptionalConnection(BaseModel):
     )
     max_object_bytes: int | None = Field(
         5242880,
-        description='Maximum bytes downloaded per blob for MIME detection and text extraction',
+        description='Maximum bytes of one object held in memory. Larger objects are streamed to a temporary file (or read by byte range where the provider supports it), so this bounds memory rather than the size of file that can be scanned. See max_file_bytes to refuse large objects outright.',
+        ge=1024,
+    )
+    max_file_bytes: int | None = Field(
+        None,
+        description='Refuse any object larger than this many bytes. 0 or unset means no limit: an object above max_object_bytes is spooled to disk rather than held in memory, so file size is bounded by free disk, not by RAM.',
+        ge=0,
+    )
+    max_archive_members: int | None = Field(
+        200,
+        description='Maximum member files expanded from one archive into child assets. Bounds fan-out, and is the guard against a zip bomb.',
+        ge=1,
+        le=10000,
+    )
+    max_archive_member_bytes: int | None = Field(
+        10485760,
+        description='Maximum uncompressed bytes read from a single archive member',
         ge=1024,
-        le=52428800,
+    )
+    max_archive_total_bytes: int | None = Field(
+        104857600,
+        description='Maximum uncompressed bytes read across all members of one archive. The decompression-ratio ceiling: a zip bomb hits this before it hits memory.',
+        ge=1024,
+    )
+    max_embedded_files: int | None = Field(
+        200,
+        description='Maximum embedded files (parquet image/audio columns, office media) expanded from one container into child assets per run',
+        ge=1,
+        le=10000,
     )
 
 
@@ -1185,9 +1237,35 @@ class GoogleCloudStorageOptionalConnection(BaseModel):
     )
     max_object_bytes: int | None = Field(
         5242880,
-        description='Maximum bytes downloaded per object for MIME detection and text extraction',
+        description='Maximum bytes of one object held in memory. Larger objects are streamed to a temporary file (or read by byte range where the provider supports it), so this bounds memory rather than the size of file that can be scanned. See max_file_bytes to refuse large objects outright.',
         ge=1024,
-        le=52428800,
+    )
+    max_file_bytes: int | None = Field(
+        None,
+        description='Refuse any object larger than this many bytes. 0 or unset means no limit: an object above max_object_bytes is spooled to disk rather than held in memory, so file size is bounded by free disk, not by RAM.',
+        ge=0,
+    )
+    max_archive_members: int | None = Field(
+        200,
+        description='Maximum member files expanded from one archive into child assets. Bounds fan-out, and is the guard against a zip bomb.',
+        ge=1,
+        le=10000,
+    )
+    max_archive_member_bytes: int | None = Field(
+        10485760,
+        description='Maximum uncompressed bytes read from a single archive member',
+        ge=1024,
+    )
+    max_archive_total_bytes: int | None = Field(
+        104857600,
+        description='Maximum uncompressed bytes read across all members of one archive. The decompression-ratio ceiling: a zip bomb hits this before it hits memory.',
+        ge=1024,
+    )
+    max_embedded_files: int | None = Field(
+        200,
+        description='Maximum embedded files (parquet image/audio columns, office media) expanded from one container into child assets per run',
+        ge=1,
+        le=10000,
     )
 
 
@@ -4712,9 +4790,8 @@ class DropboxOptionalConnection(BaseModel):
     )
     max_object_bytes: int | None = Field(
         5242880,
-        description='Maximum bytes downloaded per file for MIME detection and text extraction',
+        description='Maximum bytes of one object held in memory. Larger objects are streamed to a temporary file (or read by byte range where the provider supports it), so this bounds memory rather than the size of file that can be scanned. See max_file_bytes to refuse large objects outright.',
         ge=1024,
-        le=52428800,
     )
     request_timeout_seconds: float | None = Field(
         60,
@@ -4728,6 +4805,33 @@ class DropboxOptionalConnection(BaseModel):
         ge=0,
         le=10,
     )
+    max_file_bytes: int | None = Field(
+        None,
+        description='Refuse any object larger than this many bytes. 0 or unset means no limit: an object above max_object_bytes is spooled to disk rather than held in memory, so file size is bounded by free disk, not by RAM.',
+        ge=0,
+    )
+    max_archive_members: int | None = Field(
+        200,
+        description='Maximum member files expanded from one archive into child assets. Bounds fan-out, and is the guard against a zip bomb.',
+        ge=1,
+        le=10000,
+    )
+    max_archive_member_bytes: int | None = Field(
+        10485760,
+        description='Maximum uncompressed bytes read from a single archive member',
+        ge=1024,
+    )
+    max_archive_total_bytes: int | None = Field(
+        104857600,
+        description='Maximum uncompressed bytes read across all members of one archive. The decompression-ratio ceiling: a zip bomb hits this before it hits memory.',
+        ge=1024,
+    )
+    max_embedded_files: int | None = Field(
+        200,
+        description='Maximum embedded files (parquet image/audio columns, office media) expanded from one container into child assets per run',
+        ge=1,
+        le=10000,
+    )
 
 
 class DropboxOptionalTeam(BaseModel):
@@ -4892,9 +4996,8 @@ class HuggingFaceOptionalConnection(BaseModel):
     )
     max_object_bytes: int | None = Field(
         26214400,
-        description='Maximum bytes streamed per file for MIME detection and text extraction (default 25 MB). Larger files are read up to this cap and then the connection is dropped.',
+        description='Maximum bytes of one object held in memory. Larger objects are streamed to a temporary file (or read by byte range where the provider supports it), so this bounds memory rather than the size of file that can be scanned. See max_file_bytes to refuse large objects outright.',
         ge=1024,
-        le=104857600,
     )
     request_timeout_seconds: float | None = Field(
         60,
@@ -4908,6 +5011,33 @@ class HuggingFaceOptionalConnection(BaseModel):
         ge=0,
         le=10,
     )
+    max_file_bytes: int | None = Field(
+        None,
+        description='Refuse any object larger than this many bytes. 0 or unset means no limit: an object above max_object_bytes is spooled to disk rather than held in memory, so file size is bounded by free disk, not by RAM.',
+        ge=0,
+    )
+    max_archive_members: int | None = Field(
+        200,
+        description='Maximum member files expanded from one archive into child assets. Bounds fan-out, and is the guard against a zip bomb.',
+        ge=1,
+        le=10000,
+    )
+    max_archive_member_bytes: int | None = Field(
+        10485760,
+        description='Maximum uncompressed bytes read from a single archive member',
+        ge=1024,
+    )
+    max_archive_total_bytes: int | None = Field(
+        104857600,
+        description='Maximum uncompressed bytes read across all members of one archive. The decompression-ratio ceiling: a zip bomb hits this before it hits memory.',
+        ge=1024,
+    )
+    max_embedded_files: int | None = Field(
+        200,
+        description='Maximum embedded files (parquet image/audio columns, office media) expanded from one container into child assets per run',
+        ge=1,
+        le=10000,
+    )
 
 
 class HuggingFaceOptional(BaseModel):

{classifyre_cli-0.4.82 → classifyre_cli-0.4.83}/src/pipeline/payload_window.py

@@ -114,8 +114,16 @@ class PayloadCursor:
 
         Returning None is the restart signal. It fires on an unreadable cursor,
         an unknown format version, a checksum that no longer matches the file,
-        and a strategy switch — in each case the stored offset points at rows we
-        can no longer vouch for.
+        a strategy switch, and a completed sweep that covered no rows — in each
+        case the stored offset points at rows we can no longer vouch for.
+
+        That last one is a repair. A cursor reading ``exhausted`` with an explicit
+        ``rows_seen: 0`` was written by a run that read nothing and concluded the
+        file was finished — the signature of a payload that failed to open, not
+        one that was scanned. Such a cursor retires the asset from every future
+        scan (the scan cache skips on ``exhausted``), so it is discarded on sight
+        and the sweep starts over. A genuinely empty payload is re-read instead
+        of skipped, which costs nothing.
         """
         if not isinstance(raw, dict):
             return None
@@ -137,6 +145,15 @@ class PayloadCursor:
 
         rows_seen = raw.get("rows_seen")
         offset = _coerce_int(raw.get("offset"), 0)
+        swept_nothing = (
+            isinstance(rows_seen, int) and not isinstance(rows_seen, bool) and rows_seen <= 0
+        )
+        if bool(raw.get("exhausted", False)) and swept_nothing:
+            logger.info(
+                "Discarding a payload cursor that claims a completed sweep of 0 rows; "
+                "the payload was never actually read, so this asset restarts its sweep"
+            )
+            return None
         return cls(
             offset=max(0, offset),
             rows_seen=(max(0, int(rows_seen)) if isinstance(rows_seen, int) else None),
@@ -227,6 +244,8 @@ class PayloadWindow:
         it for free (Parquet keeps it in the footer). RANDOM uses it to seek
         straight to a random window; without it, RANDOM falls back to reservoir
         sampling, which still visits every row but holds only one window.
+        AUTOMATIC uses it to decide when a sweep is genuinely complete instead of
+        inferring it from the size of the last window (see ``_advanced``).
         """
         take = max(1, self.rows_per_page)
         unit = max(1, rows_per_unit)
@@ -244,7 +263,7 @@ class PayloadWindow:
             yield from pages(0, None)
             return
 
-        yield from self._iterate_automatic(pages, take, unit, on_cursor)
+        yield from self._iterate_automatic(pages, take, unit, on_cursor, row_count)
 
     # ── AUTOMATIC ────────────────────────────────────────────────────────
 
@@ -254,6 +273,7 @@ class PayloadWindow:
         take: int,
         unit: int,
         on_cursor: Callable[[PayloadCursor], None] | None,
+        row_count: int | None = None,
     ) -> Iterator[str]:
         start = self.start_row
         emitted = 0
@@ -276,15 +296,33 @@ class PayloadWindow:
                 yield page
 
         if on_cursor is not None:
-            on_cursor(self._advanced(start, emitted, take, unit))
+            on_cursor(self._advanced(start, emitted, take, unit, row_count))
 
-    def _advanced(self, start: int, emitted: int, take: int, unit: int) -> PayloadCursor:
+    def _advanced(
+        self,
+        start: int,
+        emitted: int,
+        take: int,
+        unit: int,
+        row_count: int | None = None,
+    ) -> PayloadCursor:
         """The cursor to persist after emitting ``emitted`` pages from ``start``.
 
-        An underfilled window means the reader ran off the end, which is what
-        marks the pass complete: the offset wraps to 0 so the next run re-reads
-        from the top, and ``exhausted`` tells the scan cache the payload has been
-        covered.
+        ``exhausted`` is the flag that lets the scan cache skip this asset for
+        good, so it is only ever set on positive proof that every row has been
+        read:
+
+        * when the reader knows the payload's row count (Parquet keeps it in the
+          footer), the sweep is complete exactly when it has reached that count.
+          Nothing is inferred, which is what stops a file whose length is a
+          multiple of ``rows_per_page`` from wrapping forever;
+        * otherwise the only available signal is an underfilled window, and it is
+          trusted *except* when the window came back completely empty from row 0.
+          A reader that produced nothing at all has not proved the payload is
+          empty — it may have failed — and treating that as a finished sweep is
+          how an unreadable file gets banked as fully scanned and skipped for
+          ever after. Re-reading a genuinely empty payload costs one cheap pass;
+          the other mistake costs the whole file.
         """
         prior = self.prior
         passes = prior.passes if prior is not None else 0
@@ -292,10 +330,18 @@ class PayloadWindow:
         # rows each, so a full window is that many pages.
         expected_units = -(-take // unit)
         reached = start + emitted * unit
-        if emitted < expected_units:
+
+        if row_count is not None and row_count >= 0:
+            complete = reached >= row_count
+            rows_seen = min(reached, row_count)
+        else:
+            complete = emitted < expected_units and not (emitted == 0 and start == 0)
+            rows_seen = reached
+
+        if complete:
             return PayloadCursor(
                 offset=0,
-                rows_seen=reached,
+                rows_seen=rows_seen,
                 passes=passes + 1,
                 exhausted=True,
                 checksum=self.checksum,
@@ -303,7 +349,7 @@ class PayloadWindow:
             )
         return PayloadCursor(
             offset=reached,
-            rows_seen=prior.rows_seen if prior is not None else None,
+            rows_seen=(rows_seen if row_count is not None else prior.rows_seen if prior else None),
             passes=passes,
             exhausted=False,
             checksum=self.checksum,

{classifyre_cli-0.4.82 → classifyre_cli-0.4.83}/src/sources/azure_blob_storage/source.py

@@ -6,17 +6,22 @@ from typing import Any
 from urllib.parse import quote
 
 from ...models.generated_input import AzureBlobStorageInput
+from ...utils.range_reader import CallableRangeReader, open_buffered
 from ..dependencies import require_module
 from ..object_storage.base import ObjectRef, ObjectStorageSourceBase
 
 logger = logging.getLogger(__name__)
 
+_STREAM_CHUNK_BYTES = 1024 * 1024
+
 
 class AzureBlobStorageSource(ObjectStorageSourceBase):
     source_type = "azure_blob_storage"
     provider_label = "AZURE_BLOB_STORAGE"
     input_model = AzureBlobStorageInput
 
+    SUPPORTS_RANGE_READS = True
+
     def _required_container(self) -> str:
         container = str(self.config.required.container).strip()
         if not container:
@@ -109,6 +114,33 @@ class AzureBlobStorageSource(ObjectStorageSourceBase):
                     content_type_hint=str(content_type_hint) if content_type_hint else None,
                 )
 
+    def _blob_client(self, key: str) -> Any:
+        container_client = self._client().get_container_client(self._required_container())
+        return container_client.get_blob_client(key)
+
+    def _stream_object(self, ref: ObjectRef) -> Iterator[bytes]:
+        """Stream the whole blob in chunks — no cap, nothing held whole."""
+        downloader = self._blob_client(ref.key).download_blob(
+            timeout=self._request_timeout_seconds()
+        )
+        yield from downloader.chunks()
+
+    def _open_object_range_reader(self, ref: ObjectRef) -> Any | None:
+        """Azure takes offset/length, so a columnar blob needs no download."""
+        if not ref.size:
+            return None
+        blob_client = self._blob_client(ref.key)
+        timeout = self._request_timeout_seconds()
+        label = f"{self._required_container()}/{ref.key}"
+
+        def _fetch(start: int, end_inclusive: int) -> bytes:
+            downloader = blob_client.download_blob(
+                offset=start, length=end_inclusive - start + 1, timeout=timeout
+            )
+            return bytes(downloader.readall())
+
+        return open_buffered(CallableRangeReader(_fetch, size=int(ref.size), label=label))
+
     def _download_object(self, ref: ObjectRef) -> tuple[bytes, str | None]:
         blob_service_client = self._client()
         container = self._required_container()

{classifyre_cli-0.4.82 → classifyre_cli-0.4.83}/src/sources/base.py

@@ -378,7 +378,7 @@ class BaseSource(ABC):
 
     def iter_asset_pages(
         self,
-        file_bytes: bytes,
+        file_bytes: bytes | Any,
         mime_type: str,
         batch_size: int = 100,
         include_column_names: bool = True,
@@ -397,6 +397,11 @@ class BaseSource(ABC):
 
         Everything else — PDFs, images, audio — has no row axis and streams
         unchanged.
+
+        ``file_bytes`` may be a seekable binary handle instead of bytes (Parquet
+        only). The window then bounds what is *transferred*, not just what is
+        decoded: a source that can serve byte ranges reads the footer and the one
+        row group this run needs out of an object it never downloads whole.
         """
         from ..utils.file_parser import count_tabular_rows, iter_file_pages
 

{classifyre_cli-0.4.82 → classifyre_cli-0.4.83}/src/sources/dropbox/source.py

@@ -28,6 +28,8 @@ from ..object_storage.base import ObjectRef, ObjectStorageSourceBase
 
 logger = logging.getLogger(__name__)
 
+_DOWNLOAD_CHUNK_BYTES = 1024 * 1024
+
 # Dropbox Paper docs (and a few other cloud-native items) carry no downloadable
 # bytes. They are exported instead; markdown keeps the text intact for detectors.
 _EXPORT_MIME_TYPES = {
@@ -587,6 +589,33 @@ class DropboxSource(ObjectStorageSourceBase):
 
     # ── download ─────────────────────────────────────────────────────────
 
+    def _stream_object(self, ref: ObjectRef) -> Iterator[bytes]:
+        """Stream a Dropbox file in chunks — no cap, nothing held whole.
+
+        Exported formats (Paper docs and friends) go through the same path: the
+        export is generated server-side and streamed back like any other file.
+        """
+        dropbox_ref = ref if isinstance(ref, DropboxObjectRef) else None
+        client = self._client_for(dropbox_ref.target if dropbox_ref else _ACCOUNT_TARGET)
+        path = dropbox_ref.file_id if dropbox_ref and dropbox_ref.file_id else ref.key
+
+        if dropbox_ref is not None and not dropbox_ref.is_downloadable:
+            export_format = dropbox_ref.export_format or "markdown"
+            _export_result, response = client.files_export(path, export_format)
+            self._exported_as_by_key[ref.key] = _EXPORT_MIME_TYPES.get(export_format, "text/plain")
+        else:
+            _metadata, response = client.files_download(path)
+
+        try:
+            yield from response.iter_content(chunk_size=_DOWNLOAD_CHUNK_BYTES)
+        finally:
+            close = getattr(response, "close", None)
+            if callable(close):
+                try:
+                    close()
+                except Exception:
+                    logger.debug("Failed to close Dropbox response body")
+
     def _download_object(self, ref: ObjectRef) -> tuple[bytes, str | None]:
         max_bytes = self._max_object_bytes()
         dropbox_ref = ref if isinstance(ref, DropboxObjectRef) else None

{classifyre_cli-0.4.82 → classifyre_cli-0.4.83}/src/sources/google_cloud_storage/source.py

@@ -6,17 +6,22 @@ from collections.abc import Iterator
 from typing import Any
 
 from ...models.generated_input import GoogleCloudStorageInput
+from ...utils.range_reader import CallableRangeReader, open_buffered
 from ..dependencies import require_module
 from ..object_storage.base import ObjectRef, ObjectStorageSourceBase
 
 logger = logging.getLogger(__name__)
 
+_STREAM_CHUNK_BYTES = 1024 * 1024
+
 
 class GoogleCloudStorageSource(ObjectStorageSourceBase):
     source_type = "google_cloud_storage"
     provider_label = "GOOGLE_CLOUD_STORAGE"
     input_model = GoogleCloudStorageInput
 
+    SUPPORTS_RANGE_READS = True
+
     def _required_bucket(self) -> str:
         bucket = str(self.config.required.bucket).strip()
         if not bucket:
@@ -98,6 +103,31 @@ class GoogleCloudStorageSource(ObjectStorageSourceBase):
                 content_type_hint=str(getattr(blob, "content_type", "") or "") or None,
             )
 
+    def _stream_object(self, ref: ObjectRef) -> Iterator[bytes]:
+        """Stream the whole blob in chunks — no cap, nothing held whole."""
+        blob = self._client().bucket(self._required_bucket()).blob(ref.key)
+        with blob.open("rb") as handle:
+            while True:
+                chunk = handle.read(_STREAM_CHUNK_BYTES)
+                if not chunk:
+                    return
+                yield chunk
+
+    def _open_object_range_reader(self, ref: ObjectRef) -> Any | None:
+        """GCS takes an inclusive byte range, so a columnar blob needs no download."""
+        if not ref.size:
+            return None
+        bucket_name = self._required_bucket()
+        blob = self._client().bucket(bucket_name).blob(ref.key)
+        timeout = self._request_timeout_seconds()
+
+        def _fetch(start: int, end_inclusive: int) -> bytes:
+            return bytes(blob.download_as_bytes(start=start, end=end_inclusive, timeout=timeout))
+
+        return open_buffered(
+            CallableRangeReader(_fetch, size=int(ref.size), label=f"gs://{bucket_name}/{ref.key}")
+        )
+
     def _download_object(self, ref: ObjectRef) -> tuple[bytes, str | None]:
         client = self._client()
         bucket_name = self._required_bucket()

{classifyre_cli-0.4.82 → classifyre_cli-0.4.83}/src/sources/hugging_face/source.py

@@ -12,6 +12,7 @@ import requests
 
 from ...models.generated_input import HuggingFaceInput
 from ...models.generated_single_asset_scan_results import SingleAssetScanResults
+from ...utils.range_reader import HttpRangeReader, open_buffered
 from ..dependencies import require_module
 from ..object_storage.base import ObjectRef, ObjectStorageSourceBase
 
@@ -59,6 +60,9 @@ class HuggingFaceSource(ObjectStorageSourceBase):
     provider_label = "HUGGING_FACE"
     input_model = HuggingFaceInput
 
+    # The Hub's resolve endpoint and its CDN both serve byte ranges.
+    SUPPORTS_RANGE_READS = True
+
     def __init__(
         self,
         recipe: dict[str, Any],
@@ -409,6 +413,66 @@ class HuggingFaceSource(ObjectStorageSourceBase):
             )
         return file_bytes, content_type
 
+    def _stream_object(self, ref: ObjectRef) -> Iterator[bytes]:
+        """Stream a Hub file in chunks — no cap, nothing held whole.
+
+        ``_open_object`` spools this, so the peak cost of a file the range reader
+        cannot help with (a PDF, a zip) is a temp file rather than its full size
+        in memory.
+        """
+        response = self._session().get(
+            self._resolve_url(ref.key),
+            headers={
+                "Authorization": f"Bearer {self._token()}",
+                "User-Agent": "classifyre",
+            },
+            stream=True,
+            timeout=self._request_timeout_seconds(),
+            allow_redirects=True,
+        )
+        response.raise_for_status()
+        try:
+            yield from response.iter_content(chunk_size=_DOWNLOAD_CHUNK_BYTES)
+        finally:
+            close = getattr(response, "close", None)
+            if callable(close):
+                try:
+                    close()
+                except Exception:
+                    logger.debug("Failed to close Hugging Face response body")
+
+    def _open_object_range_reader(self, ref: ObjectRef) -> Any | None:
+        """A seekable handle over a Hub file, served by HTTP range requests.
+
+        This is what makes a repository of multi-hundred-megabyte Parquet shards
+        scannable at all. ``max_object_bytes`` bounds a whole-file download, and
+        a Parquet file cut off at that bound has lost its footer — the index of
+        where the row groups are — so it yields not fewer rows but none. Reading
+        by range inverts the cost: the sampling window picks the rows, and only
+        the footer plus the row groups holding those rows ever cross the wire.
+
+        The Hub's ``resolve`` endpoint and its CDN both honour ``Range``. If a
+        deployment ever does not, ``HttpRangeReader`` still returns correct bytes
+        (it slices a full response) and says so in the log.
+        """
+        if not ref.size:
+            return None
+
+        reader = HttpRangeReader(
+            self._session(),
+            self._resolve_url(ref.key),
+            size=int(ref.size),
+            # As in _download_object: requests drops this on the redirect to the
+            # pre-signed CDN host, which is exactly what the Hub expects.
+            headers={
+                "Authorization": f"Bearer {self._token()}",
+                "User-Agent": "classifyre",
+            },
+            timeout=self._request_timeout_seconds(),
+            label=f"hugging_face:{ref.key}",
+        )
+        return open_buffered(reader)
+
     @staticmethod
     def _read_capped(response: Any, max_bytes: int) -> bytes:
         """Read at most ``max_bytes`` + 1 bytes, then drop the connection.

{classifyre_cli-0.4.82 → classifyre_cli-0.4.83}/src/sources/local_folder/source.py

@@ -9,6 +9,7 @@ from typing import Any
 from urllib.parse import quote
 
 from ...models.generated_input import LocalFolderInput
+from ...utils.file_parser import resolve_mime_type
 from ..object_storage.base import ObjectRef, ObjectStorageSourceBase
 
 logger = logging.getLogger(__name__)
@@ -133,6 +134,27 @@ class LocalFolderSource(ObjectStorageSourceBase):
     def _list_objects(self) -> Iterator[ObjectRef]:
         yield from self._walk(self._root(), depth=0)
 
+    def _open_object(self, ref: ObjectRef) -> tuple[Any, str]:
+        """Open the file where it already is.
+
+        The one source that needs neither a download nor a spool: the payload is
+        a local file, so the parser reads it in place at any size. ``max_file_bytes``
+        keeps governing how much is pulled into memory by the *bytes* path
+        (binary detectors), not how large a file can be read.
+        """
+        file_path = self._root() / ref.key
+        handle = open(file_path, "rb")
+        try:
+            mime_type = resolve_mime_type(
+                handle,
+                declared_mime_type=ref.content_type_hint or "",
+                file_name=self._object_file_name(ref),
+            )
+        except Exception:
+            handle.close()
+            raise
+        return handle, mime_type
+
     def _download_object(self, ref: ObjectRef) -> tuple[bytes, str | None]:
         file_path = self._root() / ref.key
         max_bytes = self._max_file_bytes()