classifyre-cli 0.4.90__tar.gz → 0.4.91__tar.gz

{classifyre_cli-0.4.90 → classifyre_cli-0.4.91}/.turbo/turbo-build.log

@@ -1,3 +1,3 @@
 $ uv sync
-Resolved 307 packages in 235ms
+Resolved 307 packages in 249ms
 Checked 51 packages in 1ms

{classifyre_cli-0.4.90 → classifyre_cli-0.4.91}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: classifyre-cli
-Version: 0.4.90
+Version: 0.4.91
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.4.90 → classifyre_cli-0.4.91}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.4.90",
+  "version": "0.4.91",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.4.90 → classifyre_cli-0.4.91}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.4.90"
+version = "0.4.91"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 requires-python = ">=3.12"

{classifyre_cli-0.4.90 → classifyre_cli-0.4.91}/src/file_evaluation/runner.py

@@ -148,7 +148,7 @@ class FileEvaluationRunner:
                     file_mime,
                 )
 
-        # Files that embed whole other files (parquet file columns, office media)
+        # Files that embed whole other files (parquet/Arrow file columns, office media)
         # get each embedded file run through the binary detectors directly, with
         # findings tagged by the embedded location so the UI can group them.
         embedded_files = has_embedded_files(mime_type)

{classifyre_cli-0.4.90 → classifyre_cli-0.4.91}/src/sources/atlassian_common.py

@@ -11,7 +11,7 @@ from urllib.parse import urljoin, urlsplit, urlunsplit
 
 import requests
 
-from ..utils.file_parser import normalize_mime_type
+from ..utils.file_parser import ARROW_MIME_TYPES, normalize_mime_type
 from ..utils.hashing import normalize_http_url
 
 logger = logging.getLogger(__name__)
@@ -25,6 +25,7 @@ TABULAR_MIME_TYPES = {
     "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
     "application/parquet",
     "application/vnd.apache.parquet",
+    *ARROW_MIME_TYPES,
 }
 
 TABULAR_FILE_EXTENSIONS = {
@@ -33,6 +34,10 @@ TABULAR_FILE_EXTENSIONS = {
     ".xls",
     ".xlsx",
     ".parquet",
+    ".arrow",
+    ".arrows",
+    ".feather",
+    ".ipc",
 }
 
 

{classifyre_cli-0.4.90 → classifyre_cli-0.4.91}/src/sources/base.py

@@ -399,7 +399,7 @@ class BaseSource(ABC):
         unchanged.
 
         ``file_bytes`` may be a seekable binary handle instead of bytes (Parquet
-        only). The window then bounds what is *transferred*, not just what is
+        and Arrow). The window then bounds what is *transferred*, not just what is
         decoded: a source that can serve byte ranges reads the footer and the one
         row group this run needs out of an object it never downloads whole.
         """
@@ -432,11 +432,11 @@ class BaseSource(ABC):
     def _payload_rows_per_page_unit(mime_type: str, batch_size: int) -> int:
         """How many payload rows one page from ``iter_file_pages`` carries.
 
-        Record-shaped readers (Parquet, CSV/TSV) emit a page per row. Spreadsheets
-        reach the detectors as extracted text, which is paged in blocks of
-        ``batch_size`` lines — one line per sheet row.
+        Record-shaped readers (Parquet, Arrow IPC, CSV/TSV) emit a page per row.
+        Spreadsheets reach the detectors as extracted text, which is paged in
+        blocks of ``batch_size`` lines — one line per sheet row.
         """
-        from ..utils.file_parser import normalize_mime_type
+        from ..utils.file_parser import ARROW_MIME_TYPES, normalize_mime_type
 
         normalized = normalize_mime_type(mime_type)
         record_shaped = {
@@ -444,6 +444,7 @@ class BaseSource(ABC):
             "application/vnd.apache.parquet",
             "text/csv",
             "text/tab-separated-values",
+            *ARROW_MIME_TYPES,
         }
         return 1 if normalized in record_shaped else max(1, batch_size)
 

{classifyre_cli-0.4.90 → classifyre_cli-0.4.91}/src/sources/confluence/source.py

@@ -71,6 +71,10 @@ FILE_EXTENSION_HINTS: dict[str, OutputAssetType] = {
     ".tar": OutputAssetType.BINARY,
     ".gz": OutputAssetType.BINARY,
     ".parquet": OutputAssetType.TABLE,
+    ".arrow": OutputAssetType.TABLE,
+    ".arrows": OutputAssetType.TABLE,
+    ".feather": OutputAssetType.TABLE,
+    ".ipc": OutputAssetType.TABLE,
     ".json": OutputAssetType.TXT,
     ".xml": OutputAssetType.TXT,
     ".txt": OutputAssetType.TXT,

{classifyre_cli-0.4.90 → classifyre_cli-0.4.91}/src/sources/email/source.py

@@ -25,7 +25,11 @@ from ...models.generated_single_asset_scan_results import (
 )
 from ...utils.content_extraction import html_to_text
 from ...utils.file_metadata import extract_file_metadata
-from ...utils.file_parser import normalize_mime_type, resolve_mime_type
+from ...utils.file_parser import (
+    ARROW_MIME_TYPES,
+    normalize_mime_type,
+    resolve_mime_type,
+)
 from ...utils.hashing import hash_id
 from ..base import BaseSource
 from ..dependencies import require_module
@@ -39,6 +43,7 @@ _TABULAR_MIME_TYPES = {
     "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
     "application/parquet",
     "application/vnd.apache.parquet",
+    *ARROW_MIME_TYPES,
 }
 
 # MIME types whose text can be extracted (by file_parser) and so should route

{classifyre_cli-0.4.90 → classifyre_cli-0.4.91}/src/sources/google_workspace/source.py

@@ -28,7 +28,11 @@ from ...models.generated_single_asset_scan_results import (
     SingleAssetScanResults,
 )
 from ...utils.file_metadata import extract_file_metadata
-from ...utils.file_parser import normalize_mime_type, resolve_mime_type
+from ...utils.file_parser import (
+    ARROW_MIME_TYPES,
+    normalize_mime_type,
+    resolve_mime_type,
+)
 from ...utils.hashing import hash_id
 from ..base import BaseSource
 from ..dependencies import require_module
@@ -85,6 +89,7 @@ _TABULAR_MIME_TYPES = {
     "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
     "application/parquet",
     "application/vnd.apache.parquet",
+    *ARROW_MIME_TYPES,
 }
 
 _FILE_EXTENSION_HINTS: dict[str, OutputAssetType] = {
@@ -109,6 +114,10 @@ _FILE_EXTENSION_HINTS: dict[str, OutputAssetType] = {
     ".xlsx": OutputAssetType.TABLE,
     ".pptx": OutputAssetType.BINARY,
     ".parquet": OutputAssetType.TABLE,
+    ".arrow": OutputAssetType.TABLE,
+    ".arrows": OutputAssetType.TABLE,
+    ".feather": OutputAssetType.TABLE,
+    ".ipc": OutputAssetType.TABLE,
     ".json": OutputAssetType.TXT,
     ".xml": OutputAssetType.TXT,
     ".txt": OutputAssetType.TXT,

{classifyre_cli-0.4.90 → classifyre_cli-0.4.91}/src/sources/hugging_face/source.py

@@ -450,6 +450,9 @@ class HuggingFaceSource(ObjectStorageSourceBase):
         where the row groups are — so it yields not fewer rows but none. Reading
         by range inverts the cost: the sampling window picks the rows, and only
         the footer plus the row groups holding those rows ever cross the wire.
+        Arrow IPC shards are read the same way — their footer indexes batches by
+        offset rather than by row, so a window costs the batches up to its end
+        instead of the whole shard.
 
         The Hub's ``resolve`` endpoint and its CDN both honour ``Range``. If a
         deployment ever does not, ``HttpRangeReader`` still returns correct bytes

{classifyre_cli-0.4.90 → classifyre_cli-0.4.91}/src/sources/microsoft_365/source.py

@@ -32,7 +32,11 @@ from ...models.generated_single_asset_scan_results import (
     SingleAssetScanResults,
 )
 from ...utils.file_metadata import extract_file_metadata
-from ...utils.file_parser import normalize_mime_type, resolve_mime_type
+from ...utils.file_parser import (
+    ARROW_MIME_TYPES,
+    normalize_mime_type,
+    resolve_mime_type,
+)
 from ...utils.hashing import hash_id
 from ..base import BaseSource
 from ..dependencies import require_module
@@ -53,6 +57,7 @@ _TABULAR_MIME_TYPES = {
     "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
     "application/parquet",
     "application/vnd.apache.parquet",
+    *ARROW_MIME_TYPES,
 }
 
 _FILE_EXTENSION_HINTS: dict[str, OutputAssetType] = {
@@ -77,6 +82,10 @@ _FILE_EXTENSION_HINTS: dict[str, OutputAssetType] = {
     ".xlsx": OutputAssetType.TABLE,
     ".pptx": OutputAssetType.BINARY,
     ".parquet": OutputAssetType.TABLE,
+    ".arrow": OutputAssetType.TABLE,
+    ".arrows": OutputAssetType.TABLE,
+    ".feather": OutputAssetType.TABLE,
+    ".ipc": OutputAssetType.TABLE,
     ".json": OutputAssetType.TXT,
     ".xml": OutputAssetType.TXT,
     ".txt": OutputAssetType.TXT,

{classifyre_cli-0.4.90 → classifyre_cli-0.4.91}/src/sources/notion/source.py

@@ -68,6 +68,10 @@ FILE_EXTENSION_HINTS: dict[str, OutputAssetType] = {
     ".xls": OutputAssetType.TABLE,
     ".xlsx": OutputAssetType.TABLE,
     ".parquet": OutputAssetType.TABLE,
+    ".arrow": OutputAssetType.TABLE,
+    ".arrows": OutputAssetType.TABLE,
+    ".feather": OutputAssetType.TABLE,
+    ".ipc": OutputAssetType.TABLE,
     ".csv": OutputAssetType.TABLE,
     ".tsv": OutputAssetType.TABLE,
     ".json": OutputAssetType.TXT,

{classifyre_cli-0.4.90 → classifyre_cli-0.4.91}/src/sources/object_storage/base.py

@@ -38,6 +38,8 @@ from ...utils.embedded_files import (
 )
 from ...utils.file_metadata import extract_file_metadata
 from ...utils.file_parser import (
+    ARROW_FILE_MIME_TYPE,
+    ARROW_MIME_TYPES,
     count_tabular_rows,
     infer_mime_type_from_file_name,
     normalize_mime_type,
@@ -66,6 +68,7 @@ _TABULAR_MIME_TYPES = {
     "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
     "application/parquet",
     "application/vnd.apache.parquet",
+    *ARROW_MIME_TYPES,
 }
 
 
@@ -109,6 +112,10 @@ _FILE_EXTENSION_HINTS: dict[str, OutputAssetType] = {
     ".gz": OutputAssetType.BINARY,
     ".tgz": OutputAssetType.BINARY,
     ".parquet": OutputAssetType.TABLE,
+    ".arrow": OutputAssetType.TABLE,
+    ".arrows": OutputAssetType.TABLE,
+    ".feather": OutputAssetType.TABLE,
+    ".ipc": OutputAssetType.TABLE,
     ".json": OutputAssetType.TXT,
     ".xml": OutputAssetType.TXT,
     ".txt": OutputAssetType.TXT,
@@ -149,8 +156,12 @@ _MAX_PENDING_CONTAINER_BYTES = 256 * 1024 * 1024
 
 # Formats a seekable handle buys anything for: the reader can seek to an index
 # and pull only the parts it needs. Everything else is extracted whole, so a
-# handle would just be a slower way to download the object.
-_RANGE_READABLE_MIME_TYPES = frozenset({"application/parquet", "application/vnd.apache.parquet"})
+# handle would just be a slower way to download the object. The Arrow *streaming*
+# layout is deliberately absent: it has no index, so reading it through ranges
+# transfers the same bytes in more requests.
+_RANGE_READABLE_MIME_TYPES = frozenset(
+    {"application/parquet", "application/vnd.apache.parquet", ARROW_FILE_MIME_TYPE}
+)
 
 
 @dataclass

{classifyre_cli-0.4.90 → classifyre_cli-0.4.91}/src/utils/embedded_files.py

@@ -1,20 +1,21 @@
-"""Extract whole files embedded inside parquet rows and OOXML media folders.
+"""Extract whole files embedded inside table rows and OOXML media folders.
 
 Some files carry other *files* inside them rather than beside them: a HuggingFace
 dataset stores an ``image`` or ``audio`` column as ``struct<bytes, path>``, a
-parquet export can hold a column of PDF or archive blobs, and Office documents
-embed media under their ``media/`` folders. This module surfaces those as raw
-bytes so the scan pipeline can turn each one into its own child asset — parsed by
-``utils.file_parser`` and scanned like any standalone file — instead of dumping
-undecodable bytes into text detectors.
-
-Parquet extraction is row-addressable. ``start_row``/``max_rows`` bound it to the
-same window the sampling strategy applies to that payload's rows (see
-``pipeline.payload_window``), so a run materializes the children of the rows it
-actually scanned rather than the first N rows of the file on every run. Row
-numbers stay absolute, which is what keeps a child's identity stable: the image in
-row 4,000,001 is ``row=4000001;col=image`` whichever window found it. OOXML media
-has no row axis, so the bounds are ignored there.
+parquet or Arrow export can hold a column of PDF or archive blobs, and Office
+documents embed media under their ``media/`` folders. This module surfaces those
+as raw bytes so the scan pipeline can turn each one into its own child asset —
+parsed by ``utils.file_parser`` and scanned like any standalone file — instead of
+dumping undecodable bytes into text detectors.
+
+Row-shaped containers (parquet, Arrow IPC / Feather) are extracted by row.
+``start_row``/``max_rows`` bound the walk to the same window the sampling strategy
+applies to that payload's rows (see ``pipeline.payload_window``), so a run
+materializes the children of the rows it actually scanned rather than the first N
+rows of the file on every run. Row numbers stay absolute, which is what keeps a
+child's identity stable: the image in row 4,000,001 is ``row=4000001;col=image``
+whichever window found it, and whichever format it was stored in. OOXML media has
+no row axis, so the bounds are ignored there.
 
 On any missing optional dependency or parse failure the iterators log a warning and
 yield nothing, so callers degrade gracefully.
@@ -30,11 +31,14 @@ from dataclasses import dataclass
 from typing import Any
 
 from .file_parser import (
+    ARROW_MIME_TYPES,
     OCTET_STREAM,
+    _arrow_truncation_error,
     _normalize_mime_type,
     _parquet_row_group_span,
     _require_file_processing,
     is_readable_text,
+    open_arrow_batches,
     resolve_mime_type,
 )
 
@@ -121,7 +125,11 @@ def classify_embedded_mime(mime_type: str) -> str:
 def has_embedded_files(mime_type: str) -> bool:
     """Return True if this MIME type is a container we can pull whole files out of."""
     normalized = _normalize_mime_type(mime_type)
-    return normalized in _PARQUET_MIME_TYPES or normalized in _OOXML_MIME_TYPES
+    return (
+        normalized in _PARQUET_MIME_TYPES
+        or normalized in ARROW_MIME_TYPES
+        or normalized in _OOXML_MIME_TYPES
+    )
 
 
 def iter_embedded_files(
@@ -132,14 +140,14 @@ def iter_embedded_files(
     max_rows: int | None = None,
     max_files: int = DEFAULT_MAX_EMBEDDED_FILES,
 ) -> Iterator[EmbeddedFile]:
-    """Yield the files embedded in a parquet or OOXML container.
+    """Yield the files embedded in a parquet, Arrow IPC, or OOXML container.
 
-    ``start_row``/``max_rows`` bound a parquet walk to one row window and are
+    ``start_row``/``max_rows`` bound a row-shaped walk to one row window and are
     ignored for containers with no row axis.
 
-    ``source`` may be a seekable handle rather than bytes (parquet only), which
-    is how an object too large to download still yields its embedded children:
-    only the row groups the window covers are transferred. OOXML has no row axis
+    ``source`` may be a seekable handle rather than bytes (parquet and Arrow),
+    which is how an object too large to download still yields its embedded
+    children: only the batches the window covers are read. OOXML has no row axis
     and is read whole, so it stays on bytes.
     """
     is_bytes = isinstance(source, bytes | bytearray)
@@ -153,6 +161,13 @@ def iter_embedded_files(
             max_rows=max_rows,
             max_files=max_files,
         )
+    elif normalized in ARROW_MIME_TYPES:
+        yield from _iter_arrow_files(
+            source,
+            start_row=start_row,
+            max_rows=max_rows,
+            max_files=max_files,
+        )
     elif normalized in _OOXML_MIME_TYPES and is_bytes:
         yield from _iter_ooxml_files(source, max_files)
 
@@ -188,33 +203,10 @@ def embedded_cell_name(cell: object, kind: str) -> str:
     return ""
 
 
-def detect_parquet_payload_columns(
-    parquet_file: Any,
-    *,
-    sample_row_group: int | None = None,
-) -> dict[str, EmbeddedColumn]:
-    """Map column name → ``EmbeddedColumn`` for every column whose cells hold bytes.
-
-    All of them are returned, classified by what the bytes turned out to be, because
-    each class needs different handling and none of them may fall through to
-    ``str(cell)`` — that renders a Python bytes repr, which is how a column of JSON
-    documents reached the detectors as ``b'{"email": ...}'`` with escaped quotes.
-
-    Classification is per column, sampled once: sniffing every cell of a
-    million-row file would cost more than the scan it feeds. A column of uniform
-    content — the normal case — is therefore classified correctly, and the
-    child-asset path re-checks each cell it emits anyway.
-
-    Returns ``{}`` when pyarrow is unavailable or no column carries bytes.
-    """
-    try:
-        pa = _require_file_processing("pyarrow")
-    except Exception as exc:  # pragma: no cover - dependency missing
-        logger.warning("Cannot inspect parquet file columns: %s", exc)
-        return {}
-
+def _byte_carrying_columns(pa: Any, schema: Any) -> list[tuple[str, str]]:
+    """``(column name, kind)`` for every field of an Arrow schema holding bytes."""
     candidates: list[tuple[str, str]] = []
-    for field in parquet_file.schema_arrow:
+    for field in schema:
         field_type = field.type
         if pa.types.is_struct(field_type):  # type: ignore[attr-defined]
             child_names = {
@@ -227,15 +219,31 @@ def detect_parquet_payload_columns(
             field_type
         ):
             candidates.append((field.name, "binary"))
+    return candidates
 
-    if not candidates:
+
+def detect_parquet_payload_columns(
+    parquet_file: Any,
+    *,
+    sample_row_group: int | None = None,
+) -> dict[str, EmbeddedColumn]:
+    """``detect_payload_columns`` for a Parquet file, sampled from one row group.
+
+    Naming the row group matters: handed an open-ended batch iterator, pyarrow
+    buffers every group in the file before yielding the first one — which over a
+    range-reading handle means downloading the whole object to sniff one batch.
+    Callers that already know which group they are about to read pass it, so the
+    sample costs nothing beyond the window itself.
+    """
+    try:
+        pa = _require_file_processing("pyarrow")
+    except Exception as exc:  # pragma: no cover - dependency missing
+        logger.warning("Cannot inspect parquet file columns: %s", exc)
+        return {}
+
+    if not _byte_carrying_columns(pa, parquet_file.schema_arrow):
         return {}
 
-    # Naming the row group matters: handed an open-ended batch iterator, pyarrow
-    # buffers every group in the file before yielding the first one — which over a
-    # range-reading handle means downloading the whole object to sniff one batch.
-    # Callers that already know which group they are about to read pass it, so the
-    # sample costs nothing beyond the window itself.
     try:
         sample_kwargs: dict[str, Any] = {"batch_size": _COLUMN_SAMPLE_ROWS}
         groups = int(parquet_file.metadata.num_row_groups)
@@ -247,6 +255,34 @@ def detect_parquet_payload_columns(
         logger.warning("Cannot sample parquet columns: %s", exc)
         return {}
 
+    return detect_payload_columns(parquet_file.schema_arrow, sample)
+
+
+def detect_payload_columns(schema: Any, sample: Any) -> dict[str, EmbeddedColumn]:
+    """Map column name → ``EmbeddedColumn`` for every column whose cells hold bytes.
+
+    All of them are returned, classified by what the bytes turned out to be, because
+    each class needs different handling and none of them may fall through to
+    ``str(cell)`` — that renders a Python bytes repr, which is how a column of JSON
+    documents reached the detectors as ``b'{"email": ...}'`` with escaped quotes.
+
+    Classification is per column, sampled once from ``sample`` (one record batch):
+    sniffing every cell of a million-row file would cost more than the scan it
+    feeds. A column of uniform content — the normal case — is therefore classified
+    correctly, and the child-asset path re-checks each cell it emits anyway.
+
+    Returns ``{}`` when pyarrow is unavailable or no column carries bytes.
+    """
+    try:
+        pa = _require_file_processing("pyarrow")
+    except Exception as exc:  # pragma: no cover - dependency missing
+        logger.warning("Cannot inspect columns for embedded files: %s", exc)
+        return {}
+
+    candidates = _byte_carrying_columns(pa, schema)
+    if not candidates:
+        return {}
+
     columns: dict[str, EmbeddedColumn] = {}
     for name, kind in candidates:
         mime, sample_bytes = _sample_column_payload(sample, name, kind)
@@ -385,6 +421,93 @@ def _iter_parquet_files(
         logger.warning("Parquet embedded-file iteration failed: %s", exc)
 
 
+# ---------------------------------------------------------------------------
+# Arrow IPC / Feather
+# ---------------------------------------------------------------------------
+
+
+def _iter_arrow_files(
+    source: bytes | Any,
+    *,
+    start_row: int,
+    max_rows: int | None,
+    max_files: int,
+) -> Iterator[EmbeddedFile]:
+    """The parquet walk over an Arrow payload: same rows, same child identities.
+
+    A HuggingFace dataset is published as both formats, so an ``image`` column is
+    a ``struct<bytes, path>`` either way and has to become the same child asset
+    either way — ``row=N;col=image``, numbered absolutely, so a second run updates
+    the child rather than duplicating it.
+    """
+    truncation = _arrow_truncation_error(source)
+    if truncation is not None:
+        logger.warning("%s Skipping embedded-file extraction.", truncation)
+        return
+
+    try:
+        schema, batches = open_arrow_batches(source, batch_rows=_ROW_BATCH_SIZE)
+    except Exception as exc:
+        logger.warning("Cannot open Arrow payload for embedded-file extraction: %s", exc)
+        return
+
+    begin = max(0, start_row)
+    files = 0
+    rows_read = 0
+    abs_row = 0
+    # Only file-bearing columns produce child assets. A text column is decoded into
+    # its own row's text by the page iterator, so walking it here would scan the
+    # same characters twice under two different asset identities.
+    columns: dict[str, EmbeddedColumn] | None = None
+    try:
+        for batch in batches:
+            rows = batch.num_rows
+            if abs_row + rows <= begin:
+                abs_row += rows
+                continue
+            if columns is None:
+                # Sampled from the first batch of the window, which is being read
+                # anyway — the same bargain the parquet walk strikes.
+                columns = {
+                    name: column
+                    for name, column in detect_payload_columns(schema, batch).items()
+                    if column.content == CONTENT_FILE
+                }
+                if not columns:
+                    return
+            for local_index in range(rows):
+                if abs_row < begin:
+                    abs_row += 1
+                    continue
+                if max_rows is not None and rows_read >= max_rows:
+                    return
+                for column_name, column in columns.items():
+                    cell = batch.column(column_name)[local_index].as_py()
+                    raw = extract_embedded_bytes(cell, column.kind)
+                    if not raw:
+                        continue
+                    mime = resolve_mime_type(raw, file_name=embedded_cell_name(cell, column.kind))
+                    if not is_embeddable_file_mime(mime):
+                        continue
+                    yield EmbeddedFile(
+                        location=f"row={abs_row + 1};col={column_name}",
+                        file_bytes=raw,
+                        mime_type=mime,
+                    )
+                    files += 1
+                    if files >= max_files:
+                        logger.info(
+                            "Reached max embedded files (%d); stopping extraction", max_files
+                        )
+                        return
+                abs_row += 1
+                rows_read += 1
+            if max_rows is not None and rows_read >= max_rows:
+                return
+    except Exception as exc:
+        logger.warning("Arrow embedded-file iteration failed: %s", exc)
+
+
 # ---------------------------------------------------------------------------
 # OOXML (xlsx / docx / pptx)
 # ---------------------------------------------------------------------------

{classifyre_cli-0.4.90 → classifyre_cli-0.4.91}/src/utils/file_metadata.py

@@ -82,6 +82,8 @@ def extract_file_metadata(
             metadata.update(_docx_metadata(file_bytes))
         elif _is_parquet(normalized, extension):
             metadata.update(_parquet_metadata(file_bytes))
+        elif _is_arrow(normalized, extension):
+            metadata.update(_arrow_metadata(file_bytes))
         elif _is_xlsx(normalized, extension):
             metadata.update(_xlsx_metadata(file_bytes))
         elif _is_delimited(normalized, extension):
@@ -114,10 +116,17 @@ _IMAGE_EXTENSIONS = {
 _TEXT_EXTENSIONS = {".txt", ".md", ".xml", ".log", ".yaml", ".yml"}
 
 
+_ARROW_EXTENSIONS = {".arrow", ".arrows", ".feather", ".ipc"}
+
+
 def _is_parquet(mime: str, extension: str) -> bool:
     return "parquet" in mime or extension == ".parquet"
 
 
+def _is_arrow(mime: str, extension: str) -> bool:
+    return "arrow" in mime or "feather" in mime or extension in _ARROW_EXTENSIONS
+
+
 def _is_xlsx(mime: str, extension: str) -> bool:
     return (
         mime == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
@@ -189,6 +198,22 @@ def _parquet_metadata(file_bytes: bytes) -> dict[str, Any]:
     }
 
 
+def _arrow_metadata(file_bytes: bytes) -> dict[str, Any]:
+    # Arrow's footer indexes record batches by offset and does not total their
+    # rows, so unlike Parquet the count has to be walked for. One batch is held at
+    # a time and the payload is already resident here, so the walk costs decoding
+    # rather than transfer.
+    from .file_parser import open_arrow_batches
+
+    schema, batches = open_arrow_batches(io.BytesIO(file_bytes))
+    row_count = sum(batch.num_rows for batch in batches)
+    column_types = {field.name: str(field.type) for field in schema}
+    return {
+        "row_count": row_count,
+        "columns": build_columns(list(schema.names), column_types),
+    }
+
+
 def _xlsx_metadata(file_bytes: bytes) -> dict[str, Any]:
     import openpyxl  # type: ignore[import-untyped]