classifyre-cli 0.4.74__tar.gz → 0.4.75__tar.gz

{classifyre_cli-0.4.74 → classifyre_cli-0.4.75}/.turbo/turbo-build.log

@@ -1,3 +1,3 @@
 $ uv sync
-Resolved 303 packages in 170ms
+Resolved 303 packages in 208ms
 Checked 51 packages in 1ms

{classifyre_cli-0.4.74 → classifyre_cli-0.4.75}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: classifyre-cli
-Version: 0.4.74
+Version: 0.4.75
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.4.74 → classifyre_cli-0.4.75}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.4.74",
+  "version": "0.4.75",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.4.74 → classifyre_cli-0.4.75}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.4.74"
+version = "0.4.75"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 requires-python = ">=3.12"

{classifyre_cli-0.4.74 → classifyre_cli-0.4.75}/src/sources/atlassian_common.py

@@ -11,6 +11,7 @@ from urllib.parse import urljoin, urlsplit, urlunsplit
 
 import requests
 
+from ..utils.file_parser import normalize_mime_type
 from ..utils.hashing import normalize_http_url
 
 logger = logging.getLogger(__name__)
@@ -86,7 +87,7 @@ def extract_urls_from_text(text: str) -> list[str]:
 
 
 def is_tabular_mime_type(mime_type: str) -> bool:
-    normalized = mime_type.split(";", 1)[0].strip().lower()
+    normalized = normalize_mime_type(mime_type)
     return normalized in TABULAR_MIME_TYPES
 
 
@@ -206,7 +207,7 @@ class AtlassianCloudClient:
         for chunk in response.iter_content(chunk_size=8192):
             if chunk:
                 chunks.append(chunk)
-        mime = response.headers.get("Content-Type", "").split(";")[0].strip().lower()
+        mime = normalize_mime_type(response.headers.get("Content-Type", ""))
         return b"".join(chunks), mime
 
     def iter_confluence_results(

{classifyre_cli-0.4.74 → classifyre_cli-0.4.75}/src/sources/email/source.py

@@ -25,7 +25,7 @@ from ...models.generated_single_asset_scan_results import (
 )
 from ...utils.content_extraction import html_to_text
 from ...utils.file_metadata import extract_file_metadata
-from ...utils.file_parser import resolve_mime_type
+from ...utils.file_parser import normalize_mime_type, resolve_mime_type
 from ...utils.hashing import hash_id
 from ..base import BaseSource
 from ..dependencies import require_module
@@ -488,7 +488,7 @@ class EmailSource(BaseSource):
 
     @staticmethod
     def _asset_type_from_mime(mime: str) -> OutputAssetType:
-        normalized = (mime or "").split(";", 1)[0].strip().lower()
+        normalized = normalize_mime_type(mime)
         if normalized.startswith("image/"):
             return OutputAssetType.IMAGE
         if normalized.startswith("audio/"):

{classifyre_cli-0.4.74 → classifyre_cli-0.4.75}/src/sources/google_workspace/source.py

@@ -28,7 +28,7 @@ from ...models.generated_single_asset_scan_results import (
     SingleAssetScanResults,
 )
 from ...utils.file_metadata import extract_file_metadata
-from ...utils.file_parser import resolve_mime_type
+from ...utils.file_parser import normalize_mime_type, resolve_mime_type
 from ...utils.hashing import hash_id
 from ..base import BaseSource
 from ..dependencies import require_module
@@ -572,7 +572,7 @@ class GoogleWorkspaceSource(BaseSource):
     # -- Asset construction --
 
     def _asset_type_from_mime_or_key(self, mime_type: str | None, key: str) -> OutputAssetType:
-        normalized_mime = (mime_type or "").split(";", maxsplit=1)[0].strip().lower()
+        normalized_mime = normalize_mime_type(mime_type)
         extension = PurePosixPath(key).suffix.lower()
 
         if normalized_mime in _TABULAR_MIME_TYPES:

{classifyre_cli-0.4.74 → classifyre_cli-0.4.75}/src/sources/microsoft_365/source.py

@@ -32,7 +32,7 @@ from ...models.generated_single_asset_scan_results import (
     SingleAssetScanResults,
 )
 from ...utils.file_metadata import extract_file_metadata
-from ...utils.file_parser import resolve_mime_type
+from ...utils.file_parser import normalize_mime_type, resolve_mime_type
 from ...utils.hashing import hash_id
 from ..base import BaseSource
 from ..dependencies import require_module
@@ -679,7 +679,7 @@ class Microsoft365Source(BaseSource):
     # -- Asset construction --
 
     def _asset_type_from_mime_or_key(self, mime_type: str | None, key: str) -> OutputAssetType:
-        normalized_mime = (mime_type or "").split(";", maxsplit=1)[0].strip().lower()
+        normalized_mime = normalize_mime_type(mime_type)
         extension = PurePosixPath(key).suffix.lower()
 
         if normalized_mime in _TABULAR_MIME_TYPES:

{classifyre_cli-0.4.74 → classifyre_cli-0.4.75}/src/sources/notion/client.py

@@ -6,6 +6,8 @@ from typing import Any
 
 import requests
 
+from ...utils.file_parser import normalize_mime_type
+
 logger = logging.getLogger(__name__)
 
 NOTION_API_BASE = "https://api.notion.com/v1"
@@ -154,7 +156,7 @@ class NotionClient:
         for chunk in response.iter_content(chunk_size=8192):
             if chunk:
                 chunks.append(chunk)
-        mime = response.headers.get("Content-Type", "").split(";")[0].strip().lower()
+        mime = normalize_mime_type(response.headers.get("Content-Type", ""))
         return b"".join(chunks), mime
 
     def get_page(self, page_id: str) -> dict[str, Any]:

{classifyre_cli-0.4.74 → classifyre_cli-0.4.75}/src/sources/object_storage/base.py

@@ -25,7 +25,11 @@ from ...models.generated_single_asset_scan_results import (
 from ...utils.archive_extraction import ArchiveMember, is_archive_mime, iter_archive_members
 from ...utils.embedded_images import EmbeddedImage, has_embedded_images, iter_embedded_images
 from ...utils.file_metadata import extract_file_metadata
-from ...utils.file_parser import infer_mime_type_from_file_name, resolve_mime_type
+from ...utils.file_parser import (
+    infer_mime_type_from_file_name,
+    normalize_mime_type,
+    resolve_mime_type,
+)
 from ...utils.hashing import hash_id, unhash_id
 from ..base import BaseSource
 from ..dependencies import require_module
@@ -325,7 +329,7 @@ class ObjectStorageSourceBase(BaseSource, ABC):
         return ref.key
 
     def _asset_type_from_mime_or_key(self, mime_type: str | None, key: str) -> OutputAssetType:
-        normalized_mime = (mime_type or "").split(";", maxsplit=1)[0].strip().lower()
+        normalized_mime = normalize_mime_type(mime_type)
         extension = self._file_extension(key)
 
         if normalized_mime in _TABULAR_MIME_TYPES:
@@ -397,7 +401,7 @@ class ObjectStorageSourceBase(BaseSource, ABC):
 
     def _build_snapshot(self, ref: ObjectRef) -> ContentSnapshot:
         if self._discovery_only or not self._include_content_preview():
-            mime = (ref.content_type_hint or "").split(";", maxsplit=1)[0].strip().lower()
+            mime = normalize_mime_type(ref.content_type_hint)
             if not mime:
                 mime = infer_mime_type_from_file_name(self._object_file_name(ref))
             return ContentSnapshot(
@@ -426,7 +430,7 @@ class ObjectStorageSourceBase(BaseSource, ABC):
             declared_mime_type=content_type_hint or ref.content_type_hint or "",
             file_name=self._object_file_name(ref),
         )
-        normalized_mime = mime_type.split(";", 1)[0].strip().lower()
+        normalized_mime = normalize_mime_type(mime_type)
 
         # Images and audio/video are extractable automatically. Opaque binaries
         # remain available to binary detectors but have no text page stream.

{classifyre_cli-0.4.74 → classifyre_cli-0.4.75}/src/utils/file_parser.py

@@ -61,6 +61,37 @@ class ParsedBytes:
     parse_error: str | None = None
 
 
+OCTET_STREAM = "application/octet-stream"
+
+# Spellings of "I don't know what these bytes are". Only the first is the
+# registered type, but object stores are inconsistent: S3-compatible services
+# (Backblaze B2 among them) label every upload `binary/octet-stream`. Because
+# that is not the IANA spelling, treating it as a *real* content type made it
+# win over magic-byte detection and filename inference — so every object served
+# that way skipped MIME resolution entirely, matched no branch in extract_text,
+# and returned no text AND no error. Assets were then recorded as scanned and
+# empty, which is indistinguishable from a genuinely empty file.
+_GENERIC_BINARY_MIME_TYPES = frozenset(
+    {
+        OCTET_STREAM,
+        "binary/octet-stream",
+        "application/binary",
+        "application/unknown",
+        "unknown/unknown",
+    }
+)
+
+
+def _canonical_mime_type(mime_type: str) -> str:
+    """Collapse every "unknown bytes" spelling onto the registered one.
+
+    Downstream code compares against ``application/octet-stream`` in a dozen
+    places (is_binary, archive handling, detector content-type matching); mapping
+    aliases here means none of those have to know the alias list.
+    """
+    return OCTET_STREAM if mime_type in _GENERIC_BINARY_MIME_TYPES else mime_type
+
+
 _TEXT_RAW_MIME_TYPES = {
     "application/json",
     "application/xml",
@@ -327,10 +358,21 @@ def _require_file_processing(module_name: str) -> object:
     return require_module(module_name, "file parser", ["file-processing"])
 
 
-def _normalize_mime_type(mime_type: str | None) -> str:
+def normalize_mime_type(mime_type: str | None) -> str:
+    """Strip parameters, lower-case, and collapse "unknown bytes" aliases.
+
+    The single place MIME strings are normalized. Sources must use this rather
+    than an inline ``split(";")[0].strip().lower()``: that idiom looks equivalent
+    but skips the alias collapsing, which is exactly how `binary/octet-stream`
+    from S3-compatible stores slipped through as a real content type.
+    """
     if not mime_type:
         return ""
-    return str(mime_type).split(";", 1)[0].strip().lower()
+    return _canonical_mime_type(str(mime_type).split(";", 1)[0].strip().lower())
+
+
+# Internal alias kept so the many call sites in this module stay unchanged.
+_normalize_mime_type = normalize_mime_type
 
 
 def _file_extension(file_name: str) -> str:
@@ -357,7 +399,7 @@ def normalize_detected_mime_type(detected_mime_type: str, file_name: str) -> str
     mime = _normalize_mime_type(detected_mime_type)
     inferred_mime = infer_mime_type_from_file_name(file_name)
 
-    if not mime or mime == "application/octet-stream":
+    if not mime or mime == OCTET_STREAM:
         return inferred_mime
 
     if mime == "text/plain" and inferred_mime in _TABULAR_MIME_TYPES:
@@ -1015,17 +1057,17 @@ def resolve_mime_type(
     detected_mime = _normalize_mime_type(detect_mime_type(file_bytes))
     inferred_mime = infer_mime_type_from_file_name(file_name)
 
-    if declared_mime and declared_mime != "application/octet-stream":
+    if declared_mime and declared_mime != OCTET_STREAM:
         mime_type = declared_mime
-    elif detected_mime and detected_mime != "application/octet-stream":
+    elif detected_mime and detected_mime != OCTET_STREAM:
         mime_type = detected_mime
-    elif inferred_mime and inferred_mime != "application/octet-stream":
+    elif inferred_mime and inferred_mime != OCTET_STREAM:
         mime_type = inferred_mime
     else:
-        mime_type = declared_mime or detected_mime or inferred_mime or "application/octet-stream"
+        mime_type = declared_mime or detected_mime or inferred_mime or OCTET_STREAM
 
     mime_type = normalize_detected_mime_type(mime_type, file_name)
-    if mime_type == "application/octet-stream" and inferred_mime != "application/octet-stream":
+    if mime_type == OCTET_STREAM and inferred_mime != OCTET_STREAM:
         mime_type = inferred_mime
 
     # A text-like claim (declared upload header or file extension) must survive
@@ -1033,10 +1075,10 @@ def resolve_mime_type(
     # control-character garbage that poisons every text consumer downstream.
     if (
         file_bytes
-        and detected_mime == "application/octet-stream"
+        and detected_mime == OCTET_STREAM
         and (mime_type.startswith("text/") or mime_type in ("application/json", "application/xml"))
     ):
-        return "application/octet-stream"
+        return OCTET_STREAM
 
     return mime_type
 

{classifyre_cli-0.4.74 → classifyre_cli-0.4.75}/tests/utils/test_file_parser.py

@@ -665,6 +665,79 @@ class TestOleMimeResolution:
         assert resolve_mime_type(ole, file_name="blob") == "application/x-ole-storage"
 
 
+class TestGenericBinaryMimeAliases:
+    """S3-compatible stores declare `binary/octet-stream`, not the IANA spelling.
+
+    Treating that as a real content type let it outrank magic bytes and the file
+    extension, so every object from Backblaze B2 resolved to a type no extractor
+    handles — producing no text AND no error, which recorded the asset as
+    scanned-and-empty. Covers all formats, not just legacy Office.
+    """
+
+    OLE = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + b"\x00" * 512
+
+    @pytest.mark.parametrize(
+        "declared",
+        [
+            "binary/octet-stream",
+            "application/octet-stream",
+            "application/binary",
+            "application/unknown",
+            "unknown/unknown",
+            "BINARY/OCTET-STREAM",
+            "binary/octet-stream; charset=binary",
+            "",
+        ],
+    )
+    def test_generic_declaration_never_outranks_content(self, declared: str) -> None:
+        from src.utils.file_parser import resolve_mime_type
+
+        assert (
+            resolve_mime_type(self.OLE, declared_mime_type=declared, file_name="Cert.doc")
+            == "application/msword"
+        )
+
+    @pytest.mark.parametrize(
+        ("file_bytes", "file_name", "expected"),
+        [
+            (b"%PDF-1.4\n" + b"x" * 200, "a.pdf", "application/pdf"),
+            (b"\x89PNG\r\n\x1a\n" + b"\x00" * 64, "c.png", "image/png"),
+            (OLE, "b.xls", "application/vnd.ms-excel"),
+            (OLE, "d.ppt", "application/vnd.ms-powerpoint"),
+        ],
+    )
+    def test_every_format_survives_a_generic_declaration(
+        self, file_bytes: bytes, file_name: str, expected: str
+    ) -> None:
+        from src.utils.file_parser import resolve_mime_type
+
+        resolved = resolve_mime_type(
+            file_bytes, declared_mime_type="binary/octet-stream", file_name=file_name
+        )
+        assert resolved == expected
+
+    def test_normalize_mime_type_collapses_aliases(self) -> None:
+        from src.utils.file_parser import normalize_mime_type
+
+        for alias in ("binary/octet-stream", "application/binary", "unknown/unknown"):
+            assert normalize_mime_type(alias) == "application/octet-stream"
+        # A real type is left alone.
+        assert normalize_mime_type("application/pdf; charset=utf-8") == "application/pdf"
+
+    def test_real_declared_type_still_wins(self) -> None:
+        """The alias collapsing must not make us ignore genuine declarations."""
+        from src.utils.file_parser import resolve_mime_type
+
+        assert (
+            resolve_mime_type(
+                b"nothing recognisable here",
+                declared_mime_type="text/csv",
+                file_name="rows.bin",
+            )
+            == "text/csv"
+        )
+
+
 class TestRtfExtraction:
     def test_text_extracted(self) -> None:
         pytest.importorskip("striprtf")

{classifyre_cli-0.4.74 → classifyre_cli-0.4.75}/uv.lock

@@ -830,7 +830,7 @@ wheels = [
 
 [[package]]
 name = "classifyre-cli"
-version = "0.4.74"
+version = "0.4.75"
 source = { editable = "." }
 dependencies = [
     { name = "beautifulsoup4" },
@@ -1355,7 +1355,7 @@ youtube = [
 
 [[package]]
 name = "classifyre-schemas"
-version = "0.4.74"
+version = "0.4.75"
 source = { editable = "../../packages/schemas" }
 dependencies = [
     { name = "fastjsonschema" },