classifyre-cli 0.4.73__tar.gz → 0.4.75__tar.gz

{classifyre_cli-0.4.73 → classifyre_cli-0.4.75}/.turbo/turbo-build.log

@@ -1,3 +1,3 @@
 $ uv sync
-Resolved 303 packages in 168ms
+Resolved 303 packages in 208ms
 Checked 51 packages in 1ms

{classifyre_cli-0.4.73 → classifyre_cli-0.4.75}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: classifyre-cli
-Version: 0.4.73
+Version: 0.4.75
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured
@@ -35,6 +35,35 @@ uv sync --group detectors
 # or specific groups: --group secrets --group pii --group threat ...
 ```
 
+### System dependency: LibreOffice
+
+Legacy Office formats (`.doc`, `.xls`, `.ppt`) have no usable pure-Python parser,
+so `src/utils/legacy_office.py` shells out to `soffice --headless --convert-to
+docx|xlsx|pptx` and feeds the result through the normal OOXML extractors. Without
+it those files scan as *"no content available"* — a silent coverage gap, not an
+error.
+
+- **Kubernetes**: installed in the CLI image (`libreoffice-*-nogui`), so this
+  works out of the box.
+- **Desktop and local dev**: install it yourself — `brew install --cask libreoffice`
+  (macOS), `apt install libreoffice-writer libreoffice-calc libreoffice-impress`
+  (Debian/Ubuntu), or libreoffice.org on Windows. The desktop app deliberately
+  does not bundle LibreOffice: it is ~550 MB even stripped, and slimming the
+  upstream bundle breaks its code signature, which Apple Silicon punishes by
+  killing the process on launch.
+- **Non-standard install**: set `CLASSIFYRE_SOFFICE_PATH` to the binary. The CLI
+  checks that first, then `PATH`, then the standard macOS, Windows, and
+  Linux-tarball install locations.
+
+When no binary is found the file is **not** silently reported as empty. The scan
+raises an `ENGINE_UNAVAILABLE` text-extraction coverage error that fails that
+asset's text scan, lands in `scan_stats.errors` and the run's text-coverage
+counters, and carries a platform-specific message telling the user how to install
+LibreOffice.
+
+`tests/utils/test_legacy_office.py` skips its round-trip tests when no LibreOffice
+is found, so a green local run does not by itself prove this path works.
+
 ## Command Syntax
 
 Use the thin wrapper:

{classifyre_cli-0.4.73 → classifyre_cli-0.4.75}/README.md

@@ -18,6 +18,35 @@ uv sync --group detectors
 # or specific groups: --group secrets --group pii --group threat ...
 ```
 
+### System dependency: LibreOffice
+
+Legacy Office formats (`.doc`, `.xls`, `.ppt`) have no usable pure-Python parser,
+so `src/utils/legacy_office.py` shells out to `soffice --headless --convert-to
+docx|xlsx|pptx` and feeds the result through the normal OOXML extractors. Without
+it those files scan as *"no content available"* — a silent coverage gap, not an
+error.
+
+- **Kubernetes**: installed in the CLI image (`libreoffice-*-nogui`), so this
+  works out of the box.
+- **Desktop and local dev**: install it yourself — `brew install --cask libreoffice`
+  (macOS), `apt install libreoffice-writer libreoffice-calc libreoffice-impress`
+  (Debian/Ubuntu), or libreoffice.org on Windows. The desktop app deliberately
+  does not bundle LibreOffice: it is ~550 MB even stripped, and slimming the
+  upstream bundle breaks its code signature, which Apple Silicon punishes by
+  killing the process on launch.
+- **Non-standard install**: set `CLASSIFYRE_SOFFICE_PATH` to the binary. The CLI
+  checks that first, then `PATH`, then the standard macOS, Windows, and
+  Linux-tarball install locations.
+
+When no binary is found the file is **not** silently reported as empty. The scan
+raises an `ENGINE_UNAVAILABLE` text-extraction coverage error that fails that
+asset's text scan, lands in `scan_stats.errors` and the run's text-coverage
+counters, and carries a platform-specific message telling the user how to install
+LibreOffice.
+
+`tests/utils/test_legacy_office.py` skips its round-trip tests when no LibreOffice
+is found, so a green local run does not by itself prove this path works.
+
 ## Command Syntax
 
 Use the thin wrapper:

{classifyre_cli-0.4.73 → classifyre_cli-0.4.75}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.4.73",
+  "version": "0.4.75",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.4.73 → classifyre_cli-0.4.75}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.4.73"
+version = "0.4.75"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 requires-python = ">=3.12"

{classifyre_cli-0.4.73 → classifyre_cli-0.4.75}/src/outputs/rest.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+import os
 import random
 from typing import Any, Literal, cast
 from urllib.parse import urljoin
@@ -15,6 +16,18 @@ from .base import OutputRuntimeContext, OutputType
 
 logger = logging.getLogger(__name__)
 
+# Proves to the API that these write-backs come from a scan job it launched.
+# The API has no user authentication, so the ingest endpoints are restricted to
+# callers holding this shared secret; the API injects it into every CLI job's
+# environment (Kubernetes job env, or inherited from the parent process on
+# desktop). Absent locally, where the API leaves the restriction disabled.
+INTERNAL_KEY_HEADER = "X-Classifyre-Internal-Key"
+
+
+def _internal_key_headers() -> dict[str, str]:
+    key = os.environ.get("CLASSIFYRE_INTERNAL_KEY", "").strip()
+    return {INTERNAL_KEY_HEADER: key} if key else {}
+
 
 class _JitteredRetry(Retry):
     """urllib3 Retry subclass that adds ±25 % multiplicative jitter to the
@@ -164,6 +177,7 @@ class RestOutputSink:
         # a pod restart or server-side keep-alive timeout.  Each request opens
         # a fresh TCP connection, which is cheap enough for our batch cadence.
         self.session.headers.update({"Connection": "close"})
+        self.session.headers.update(_internal_key_headers())
         adapter = HTTPAdapter(max_retries=_RETRY_POLICY)
         self.session.mount("http://", adapter)
         self.session.mount("https://", adapter)

{classifyre_cli-0.4.73 → classifyre_cli-0.4.75}/src/sources/atlassian_common.py

@@ -11,6 +11,7 @@ from urllib.parse import urljoin, urlsplit, urlunsplit
 
 import requests
 
+from ..utils.file_parser import normalize_mime_type
 from ..utils.hashing import normalize_http_url
 
 logger = logging.getLogger(__name__)
@@ -86,7 +87,7 @@ def extract_urls_from_text(text: str) -> list[str]:
 
 
 def is_tabular_mime_type(mime_type: str) -> bool:
-    normalized = mime_type.split(";", 1)[0].strip().lower()
+    normalized = normalize_mime_type(mime_type)
     return normalized in TABULAR_MIME_TYPES
 
 
@@ -206,7 +207,7 @@ class AtlassianCloudClient:
         for chunk in response.iter_content(chunk_size=8192):
             if chunk:
                 chunks.append(chunk)
-        mime = response.headers.get("Content-Type", "").split(";")[0].strip().lower()
+        mime = normalize_mime_type(response.headers.get("Content-Type", ""))
         return b"".join(chunks), mime
 
     def iter_confluence_results(

{classifyre_cli-0.4.73 → classifyre_cli-0.4.75}/src/sources/email/source.py

@@ -25,7 +25,7 @@ from ...models.generated_single_asset_scan_results import (
 )
 from ...utils.content_extraction import html_to_text
 from ...utils.file_metadata import extract_file_metadata
-from ...utils.file_parser import resolve_mime_type
+from ...utils.file_parser import normalize_mime_type, resolve_mime_type
 from ...utils.hashing import hash_id
 from ..base import BaseSource
 from ..dependencies import require_module
@@ -488,7 +488,7 @@ class EmailSource(BaseSource):
 
     @staticmethod
     def _asset_type_from_mime(mime: str) -> OutputAssetType:
-        normalized = (mime or "").split(";", 1)[0].strip().lower()
+        normalized = normalize_mime_type(mime)
         if normalized.startswith("image/"):
             return OutputAssetType.IMAGE
         if normalized.startswith("audio/"):

{classifyre_cli-0.4.73 → classifyre_cli-0.4.75}/src/sources/google_workspace/source.py

@@ -28,7 +28,7 @@ from ...models.generated_single_asset_scan_results import (
     SingleAssetScanResults,
 )
 from ...utils.file_metadata import extract_file_metadata
-from ...utils.file_parser import resolve_mime_type
+from ...utils.file_parser import normalize_mime_type, resolve_mime_type
 from ...utils.hashing import hash_id
 from ..base import BaseSource
 from ..dependencies import require_module
@@ -572,7 +572,7 @@ class GoogleWorkspaceSource(BaseSource):
     # -- Asset construction --
 
     def _asset_type_from_mime_or_key(self, mime_type: str | None, key: str) -> OutputAssetType:
-        normalized_mime = (mime_type or "").split(";", maxsplit=1)[0].strip().lower()
+        normalized_mime = normalize_mime_type(mime_type)
         extension = PurePosixPath(key).suffix.lower()
 
         if normalized_mime in _TABULAR_MIME_TYPES:

{classifyre_cli-0.4.73 → classifyre_cli-0.4.75}/src/sources/microsoft_365/source.py

@@ -32,7 +32,7 @@ from ...models.generated_single_asset_scan_results import (
     SingleAssetScanResults,
 )
 from ...utils.file_metadata import extract_file_metadata
-from ...utils.file_parser import resolve_mime_type
+from ...utils.file_parser import normalize_mime_type, resolve_mime_type
 from ...utils.hashing import hash_id
 from ..base import BaseSource
 from ..dependencies import require_module
@@ -679,7 +679,7 @@ class Microsoft365Source(BaseSource):
     # -- Asset construction --
 
     def _asset_type_from_mime_or_key(self, mime_type: str | None, key: str) -> OutputAssetType:
-        normalized_mime = (mime_type or "").split(";", maxsplit=1)[0].strip().lower()
+        normalized_mime = normalize_mime_type(mime_type)
         extension = PurePosixPath(key).suffix.lower()
 
         if normalized_mime in _TABULAR_MIME_TYPES:

{classifyre_cli-0.4.73 → classifyre_cli-0.4.75}/src/sources/notion/client.py

@@ -6,6 +6,8 @@ from typing import Any
 
 import requests
 
+from ...utils.file_parser import normalize_mime_type
+
 logger = logging.getLogger(__name__)
 
 NOTION_API_BASE = "https://api.notion.com/v1"
@@ -154,7 +156,7 @@ class NotionClient:
         for chunk in response.iter_content(chunk_size=8192):
             if chunk:
                 chunks.append(chunk)
-        mime = response.headers.get("Content-Type", "").split(";")[0].strip().lower()
+        mime = normalize_mime_type(response.headers.get("Content-Type", ""))
         return b"".join(chunks), mime
 
     def get_page(self, page_id: str) -> dict[str, Any]:

{classifyre_cli-0.4.73 → classifyre_cli-0.4.75}/src/sources/object_storage/base.py

@@ -25,7 +25,11 @@ from ...models.generated_single_asset_scan_results import (
 from ...utils.archive_extraction import ArchiveMember, is_archive_mime, iter_archive_members
 from ...utils.embedded_images import EmbeddedImage, has_embedded_images, iter_embedded_images
 from ...utils.file_metadata import extract_file_metadata
-from ...utils.file_parser import infer_mime_type_from_file_name, resolve_mime_type
+from ...utils.file_parser import (
+    infer_mime_type_from_file_name,
+    normalize_mime_type,
+    resolve_mime_type,
+)
 from ...utils.hashing import hash_id, unhash_id
 from ..base import BaseSource
 from ..dependencies import require_module
@@ -325,7 +329,7 @@ class ObjectStorageSourceBase(BaseSource, ABC):
         return ref.key
 
     def _asset_type_from_mime_or_key(self, mime_type: str | None, key: str) -> OutputAssetType:
-        normalized_mime = (mime_type or "").split(";", maxsplit=1)[0].strip().lower()
+        normalized_mime = normalize_mime_type(mime_type)
         extension = self._file_extension(key)
 
         if normalized_mime in _TABULAR_MIME_TYPES:
@@ -397,7 +401,7 @@ class ObjectStorageSourceBase(BaseSource, ABC):
 
     def _build_snapshot(self, ref: ObjectRef) -> ContentSnapshot:
         if self._discovery_only or not self._include_content_preview():
-            mime = (ref.content_type_hint or "").split(";", maxsplit=1)[0].strip().lower()
+            mime = normalize_mime_type(ref.content_type_hint)
             if not mime:
                 mime = infer_mime_type_from_file_name(self._object_file_name(ref))
             return ContentSnapshot(
@@ -426,7 +430,7 @@ class ObjectStorageSourceBase(BaseSource, ABC):
             declared_mime_type=content_type_hint or ref.content_type_hint or "",
             file_name=self._object_file_name(ref),
         )
-        normalized_mime = mime_type.split(";", 1)[0].strip().lower()
+        normalized_mime = normalize_mime_type(mime_type)
 
         # Images and audio/video are extractable automatically. Opaque binaries
         # remain available to binary detectors but have no text page stream.

{classifyre_cli-0.4.73 → classifyre_cli-0.4.75}/src/utils/file_parser.py

@@ -61,6 +61,37 @@ class ParsedBytes:
     parse_error: str | None = None
 
 
+OCTET_STREAM = "application/octet-stream"
+
+# Spellings of "I don't know what these bytes are". Only the first is the
+# registered type, but object stores are inconsistent: S3-compatible services
+# (Backblaze B2 among them) label every upload `binary/octet-stream`. Because
+# that is not the IANA spelling, treating it as a *real* content type made it
+# win over magic-byte detection and filename inference — so every object served
+# that way skipped MIME resolution entirely, matched no branch in extract_text,
+# and returned no text AND no error. Assets were then recorded as scanned and
+# empty, which is indistinguishable from a genuinely empty file.
+_GENERIC_BINARY_MIME_TYPES = frozenset(
+    {
+        OCTET_STREAM,
+        "binary/octet-stream",
+        "application/binary",
+        "application/unknown",
+        "unknown/unknown",
+    }
+)
+
+
+def _canonical_mime_type(mime_type: str) -> str:
+    """Collapse every "unknown bytes" spelling onto the registered one.
+
+    Downstream code compares against ``application/octet-stream`` in a dozen
+    places (is_binary, archive handling, detector content-type matching); mapping
+    aliases here means none of those have to know the alias list.
+    """
+    return OCTET_STREAM if mime_type in _GENERIC_BINARY_MIME_TYPES else mime_type
+
+
 _TEXT_RAW_MIME_TYPES = {
     "application/json",
     "application/xml",
@@ -327,10 +358,21 @@ def _require_file_processing(module_name: str) -> object:
     return require_module(module_name, "file parser", ["file-processing"])
 
 
-def _normalize_mime_type(mime_type: str | None) -> str:
+def normalize_mime_type(mime_type: str | None) -> str:
+    """Strip parameters, lower-case, and collapse "unknown bytes" aliases.
+
+    The single place MIME strings are normalized. Sources must use this rather
+    than an inline ``split(";")[0].strip().lower()``: that idiom looks equivalent
+    but skips the alias collapsing, which is exactly how `binary/octet-stream`
+    from S3-compatible stores slipped through as a real content type.
+    """
     if not mime_type:
         return ""
-    return str(mime_type).split(";", 1)[0].strip().lower()
+    return _canonical_mime_type(str(mime_type).split(";", 1)[0].strip().lower())
+
+
+# Internal alias kept so the many call sites in this module stay unchanged.
+_normalize_mime_type = normalize_mime_type
 
 
 def _file_extension(file_name: str) -> str:
@@ -357,7 +399,7 @@ def normalize_detected_mime_type(detected_mime_type: str, file_name: str) -> str
     mime = _normalize_mime_type(detected_mime_type)
     inferred_mime = infer_mime_type_from_file_name(file_name)
 
-    if not mime or mime == "application/octet-stream":
+    if not mime or mime == OCTET_STREAM:
         return inferred_mime
 
     if mime == "text/plain" and inferred_mime in _TABULAR_MIME_TYPES:
@@ -1015,17 +1057,17 @@ def resolve_mime_type(
     detected_mime = _normalize_mime_type(detect_mime_type(file_bytes))
     inferred_mime = infer_mime_type_from_file_name(file_name)
 
-    if declared_mime and declared_mime != "application/octet-stream":
+    if declared_mime and declared_mime != OCTET_STREAM:
         mime_type = declared_mime
-    elif detected_mime and detected_mime != "application/octet-stream":
+    elif detected_mime and detected_mime != OCTET_STREAM:
         mime_type = detected_mime
-    elif inferred_mime and inferred_mime != "application/octet-stream":
+    elif inferred_mime and inferred_mime != OCTET_STREAM:
         mime_type = inferred_mime
     else:
-        mime_type = declared_mime or detected_mime or inferred_mime or "application/octet-stream"
+        mime_type = declared_mime or detected_mime or inferred_mime or OCTET_STREAM
 
     mime_type = normalize_detected_mime_type(mime_type, file_name)
-    if mime_type == "application/octet-stream" and inferred_mime != "application/octet-stream":
+    if mime_type == OCTET_STREAM and inferred_mime != OCTET_STREAM:
         mime_type = inferred_mime
 
     # A text-like claim (declared upload header or file extension) must survive
@@ -1033,10 +1075,10 @@ def resolve_mime_type(
     # control-character garbage that poisons every text consumer downstream.
     if (
         file_bytes
-        and detected_mime == "application/octet-stream"
+        and detected_mime == OCTET_STREAM
         and (mime_type.startswith("text/") or mime_type in ("application/json", "application/xml"))
     ):
-        return "application/octet-stream"
+        return OCTET_STREAM
 
     return mime_type
 

{classifyre_cli-0.4.73 → classifyre_cli-0.4.75}/src/utils/legacy_office.py

@@ -6,16 +6,25 @@ is fed back through the existing docx/xlsx/pptx extraction paths:
 
     .doc / .xls / .ppt → soffice --headless --convert-to → .docx / .xlsx / .pptx
 
-LibreOffice is an *optional system dependency*: when no ``soffice`` binary is
-found the conversion returns a structured error and callers degrade gracefully
-(the file stays a binary asset with a ``parse_error``).
+The CLI container image ships LibreOffice (the ``libreoffice-*-nogui`` Debian
+packages, which install ``/usr/bin/soffice``), so conversion works out of the box
+under Kubernetes. The desktop app deliberately does NOT bundle it — ~550 MB even
+stripped, and slimming the upstream bundle breaks its code signature — so there
+it is a system dependency the user installs.
+
+When no binary is found the file is **not** quietly reported as empty: the error
+from :func:`soffice_missing_error` propagates as an ENGINE_UNAVAILABLE text
+extraction coverage failure, which fails that asset's text scan and tells the
+user, per platform, how to install LibreOffice.
 """
 
 from __future__ import annotations
 
 import logging
+import os
 import shutil
 import subprocess
+import sys
 import tempfile
 import threading
 from functools import cache
@@ -46,13 +55,58 @@ LEGACY_OFFICE_MIME_TYPES = frozenset(_CONVERSION_TARGETS)
 
 _SOFFICE_TIMEOUT_SECONDS = 120
 
-# Non-PATH install locations checked after shutil.which().
-_SOFFICE_FALLBACK_PATHS = (
-    "/Applications/LibreOffice.app/Contents/MacOS/soffice",
-    "/usr/lib/libreoffice/program/soffice",
-    "/opt/libreoffice/program/soffice",
-    "C:\\Program Files\\LibreOffice\\program\\soffice.exe",
-)
+# How to actually get LibreOffice, per platform. The desktop app does not bundle
+# it, so this message is the entire remediation path a user gets — a generic
+# "install LibreOffice" would leave them guessing which package to install.
+_INSTALL_HINTS: dict[str, str] = {
+    "darwin": "install it with `brew install --cask libreoffice`, or from libreoffice.org",
+    "win32": "install it from https://www.libreoffice.org/download/",
+    "linux": (
+        "install it with "
+        "`apt install libreoffice-writer libreoffice-calc libreoffice-impress` "
+        "(or the equivalent for your distribution)"
+    ),
+}
+
+
+def soffice_missing_error() -> str:
+    """Actionable message for a deployment with no LibreOffice.
+
+    The word "unavailable" is load-bearing: it is what makes iter_file_pages
+    classify this as ENGINE_UNAVAILABLE rather than a per-document FAILED — the
+    difference between "this install cannot read .doc at all" and "this one file
+    is broken". test_legacy_office.py pins that mapping so the two cannot drift.
+    """
+    hint = _INSTALL_HINTS.get(sys.platform, "install LibreOffice")
+    return (
+        "LibreOffice is unavailable, so this .doc/.xls/.ppt file was NOT scanned. "
+        f"To scan legacy Office documents, {hint} — "
+        "or set CLASSIFYRE_SOFFICE_PATH to an existing soffice binary."
+    )
+
+
+# Explicit override, checked before anything else. The desktop app rebuilds a
+# minimal PATH for the processes it spawns, so an installation it discovered
+# itself is handed down through this variable rather than through PATH.
+SOFFICE_PATH_ENV = "CLASSIFYRE_SOFFICE_PATH"
+
+# Install locations to check when PATH comes up empty, keyed by sys.platform.
+# macOS and Windows always need these: their installers drop the binary inside an
+# app bundle / Program Files without touching PATH, and the packaged desktop app
+# rebuilds a minimal PATH that cannot see either. Linux only needs them for an
+# upstream tarball install — the distro packages (including the
+# libreoffice-*-nogui ones in the CLI image) all land on /usr/bin/soffice.
+_SOFFICE_FALLBACK_PATHS: dict[str, tuple[str, ...]] = {
+    "darwin": ("/Applications/LibreOffice.app/Contents/MacOS/soffice",),
+    "linux": (
+        "/opt/libreoffice/program/soffice",
+        "/usr/lib/libreoffice/program/soffice",
+    ),
+    "win32": (
+        "C:\\Program Files\\LibreOffice\\program\\soffice.exe",
+        "C:\\Program Files (x86)\\LibreOffice\\program\\soffice.exe",
+    ),
+}
 
 # One conversion at a time: each soffice launch is a full process spinning up
 # ~200 MB; serializing keeps peak memory bounded alongside detector workloads.
@@ -61,14 +115,36 @@ _conversion_lock = threading.Lock()
 
 @cache
 def find_soffice() -> str | None:
-    """Locate the LibreOffice binary, or None when it is not installed."""
+    """Locate the LibreOffice binary, or None when it is not installed.
+
+    Resolution order, most explicit first:
+
+    1. ``CLASSIFYRE_SOFFICE_PATH`` — an operator/desktop-supplied absolute path.
+    2. ``PATH`` — the CLI container, Linux hosts, and dev shells.
+    3. Platform install locations — macOS/Windows only (see the table above).
+
+    Cached: the answer cannot change within a process. Tests that manipulate the
+    environment must call ``find_soffice.cache_clear()``.
+    """
+    override = os.environ.get(SOFFICE_PATH_ENV, "").strip()
+    if override:
+        if Path(override).is_file():
+            return override
+        logger.warning(
+            "%s=%s does not point at an existing file; falling back to PATH lookup",
+            SOFFICE_PATH_ENV,
+            override,
+        )
+
     for name in ("soffice", "libreoffice"):
         path = shutil.which(name)
         if path:
             return path
-    for candidate in _SOFFICE_FALLBACK_PATHS:
-        if Path(candidate).exists():
+
+    for candidate in _SOFFICE_FALLBACK_PATHS.get(sys.platform, ()):
+        if Path(candidate).is_file():
             return candidate
+
     return None
 
 
@@ -90,12 +166,7 @@ def convert_legacy_office(
 
     soffice = find_soffice()
     if soffice is None:
-        return (
-            None,
-            target_mime,
-            "LibreOffice (soffice) not found; install LibreOffice to enable "
-            ".doc/.xls/.ppt extraction",
-        )
+        return None, target_mime, soffice_missing_error()
 
     try:
         with tempfile.TemporaryDirectory(prefix="classifyre-soffice-") as temp_dir:

{classifyre_cli-0.4.73 → classifyre_cli-0.4.75}/tests/utils/test_file_parser.py

@@ -665,6 +665,79 @@ class TestOleMimeResolution:
         assert resolve_mime_type(ole, file_name="blob") == "application/x-ole-storage"
 
 
+class TestGenericBinaryMimeAliases:
+    """S3-compatible stores declare `binary/octet-stream`, not the IANA spelling.
+
+    Treating that as a real content type let it outrank magic bytes and the file
+    extension, so every object from Backblaze B2 resolved to a type no extractor
+    handles — producing no text AND no error, which recorded the asset as
+    scanned-and-empty. Covers all formats, not just legacy Office.
+    """
+
+    OLE = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + b"\x00" * 512
+
+    @pytest.mark.parametrize(
+        "declared",
+        [
+            "binary/octet-stream",
+            "application/octet-stream",
+            "application/binary",
+            "application/unknown",
+            "unknown/unknown",
+            "BINARY/OCTET-STREAM",
+            "binary/octet-stream; charset=binary",
+            "",
+        ],
+    )
+    def test_generic_declaration_never_outranks_content(self, declared: str) -> None:
+        from src.utils.file_parser import resolve_mime_type
+
+        assert (
+            resolve_mime_type(self.OLE, declared_mime_type=declared, file_name="Cert.doc")
+            == "application/msword"
+        )
+
+    @pytest.mark.parametrize(
+        ("file_bytes", "file_name", "expected"),
+        [
+            (b"%PDF-1.4\n" + b"x" * 200, "a.pdf", "application/pdf"),
+            (b"\x89PNG\r\n\x1a\n" + b"\x00" * 64, "c.png", "image/png"),
+            (OLE, "b.xls", "application/vnd.ms-excel"),
+            (OLE, "d.ppt", "application/vnd.ms-powerpoint"),
+        ],
+    )
+    def test_every_format_survives_a_generic_declaration(
+        self, file_bytes: bytes, file_name: str, expected: str
+    ) -> None:
+        from src.utils.file_parser import resolve_mime_type
+
+        resolved = resolve_mime_type(
+            file_bytes, declared_mime_type="binary/octet-stream", file_name=file_name
+        )
+        assert resolved == expected
+
+    def test_normalize_mime_type_collapses_aliases(self) -> None:
+        from src.utils.file_parser import normalize_mime_type
+
+        for alias in ("binary/octet-stream", "application/binary", "unknown/unknown"):
+            assert normalize_mime_type(alias) == "application/octet-stream"
+        # A real type is left alone.
+        assert normalize_mime_type("application/pdf; charset=utf-8") == "application/pdf"
+
+    def test_real_declared_type_still_wins(self) -> None:
+        """The alias collapsing must not make us ignore genuine declarations."""
+        from src.utils.file_parser import resolve_mime_type
+
+        assert (
+            resolve_mime_type(
+                b"nothing recognisable here",
+                declared_mime_type="text/csv",
+                file_name="rows.bin",
+            )
+            == "text/csv"
+        )
+
+
 class TestRtfExtraction:
     def test_text_extracted(self) -> None:
         pytest.importorskip("striprtf")